diff --git a/Documentation/ABI/obsolete/sysfs-gpio b/Documentation/ABI/obsolete/sysfs-gpio index b8b0fd341c17..da1345d854b4 100644 --- a/Documentation/ABI/obsolete/sysfs-gpio +++ b/Documentation/ABI/obsolete/sysfs-gpio @@ -28,5 +28,5 @@ Description: /label ... (r/o) descriptive, not necessarily unique /ngpio ... (r/o) number of GPIOs; numbered N to N + (ngpio - 1) - This ABI is deprecated and will be removed after 2020. It is - replaced with the GPIO character device. + This ABI is obsoleted by Documentation/ABI/testing/gpio-cdev and will be + removed after 2020. diff --git a/Documentation/ABI/testing/debugfs-intel-iommu b/Documentation/ABI/testing/debugfs-intel-iommu new file mode 100644 index 000000000000..2ab8464504a9 --- /dev/null +++ b/Documentation/ABI/testing/debugfs-intel-iommu @@ -0,0 +1,276 @@ +What: /sys/kernel/debug/iommu/intel/iommu_regset +Date: December 2023 +Contact: Jingqi Liu +Description: + This file dumps all the register contents for each IOMMU device. + + Example in Kabylake: + + :: + + $ sudo cat /sys/kernel/debug/iommu/intel/iommu_regset + + IOMMU: dmar0 Register Base Address: 26be37000 + + Name Offset Contents + VER 0x00 0x0000000000000010 + GCMD 0x18 0x0000000000000000 + GSTS 0x1c 0x00000000c7000000 + FSTS 0x34 0x0000000000000000 + FECTL 0x38 0x0000000000000000 + + [...] + + IOMMU: dmar1 Register Base Address: fed90000 + + Name Offset Contents + VER 0x00 0x0000000000000010 + GCMD 0x18 0x0000000000000000 + GSTS 0x1c 0x00000000c7000000 + FSTS 0x34 0x0000000000000000 + FECTL 0x38 0x0000000000000000 + + [...] + + IOMMU: dmar2 Register Base Address: fed91000 + + Name Offset Contents + VER 0x00 0x0000000000000010 + GCMD 0x18 0x0000000000000000 + GSTS 0x1c 0x00000000c7000000 + FSTS 0x34 0x0000000000000000 + FECTL 0x38 0x0000000000000000 + + [...] + +What: /sys/kernel/debug/iommu/intel/ir_translation_struct +Date: December 2023 +Contact: Jingqi Liu +Description: + This file dumps the table entries for Interrupt + remapping and Interrupt posting. + + Example in Kabylake: + + :: + + $ sudo cat /sys/kernel/debug/iommu/intel/ir_translation_struct + + Remapped Interrupt supported on IOMMU: dmar0 + IR table address:100900000 + + Entry SrcID DstID Vct IRTE_high IRTE_low + 0 00:0a.0 00000080 24 0000000000040050 000000800024000d + 1 00:0a.0 00000001 ef 0000000000040050 0000000100ef000d + + Remapped Interrupt supported on IOMMU: dmar1 + IR table address:100300000 + Entry SrcID DstID Vct IRTE_high IRTE_low + 0 00:02.0 00000002 26 0000000000040010 000000020026000d + + [...] + + **** + + Posted Interrupt supported on IOMMU: dmar0 + IR table address:100900000 + Entry SrcID PDA_high PDA_low Vct IRTE_high IRTE_low + +What: /sys/kernel/debug/iommu/intel/dmar_translation_struct +Date: December 2023 +Contact: Jingqi Liu +Description: + This file dumps Intel IOMMU DMA remapping tables, such + as root table, context table, PASID directory and PASID + table entries in debugfs. For legacy mode, it doesn't + support PASID, and hence PASID field is defaulted to + '-1' and other PASID related fields are invalid. + + Example in Kabylake: + + :: + + $ sudo cat /sys/kernel/debug/iommu/intel/dmar_translation_struct + + IOMMU dmar1: Root Table Address: 0x103027000 + B.D.F Root_entry + 00:02.0 0x0000000000000000:0x000000010303e001 + + Context_entry + 0x0000000000000102:0x000000010303f005 + + PASID PASID_table_entry + -1 0x0000000000000000:0x0000000000000000:0x0000000000000000 + + IOMMU dmar0: Root Table Address: 0x103028000 + B.D.F Root_entry + 00:0a.0 0x0000000000000000:0x00000001038a7001 + + Context_entry + 0x0000000000000000:0x0000000103220e7d + + PASID PASID_table_entry + 0 0x0000000000000000:0x0000000000800002:0x00000001038a5089 + + [...] + +What: /sys/kernel/debug/iommu/intel/invalidation_queue +Date: December 2023 +Contact: Jingqi Liu +Description: + This file exports invalidation queue internals of each + IOMMU device. + + Example in Kabylake: + + :: + + $ sudo cat /sys/kernel/debug/iommu/intel/invalidation_queue + + Invalidation queue on IOMMU: dmar0 + Base: 0x10022e000 Head: 20 Tail: 20 + Index qw0 qw1 qw2 + 0 0000000000000014 0000000000000000 0000000000000000 + 1 0000000200000025 0000000100059c04 0000000000000000 + 2 0000000000000014 0000000000000000 0000000000000000 + + qw3 status + 0000000000000000 0000000000000000 + 0000000000000000 0000000000000000 + 0000000000000000 0000000000000000 + + [...] + + Invalidation queue on IOMMU: dmar1 + Base: 0x10026e000 Head: 32 Tail: 32 + Index qw0 qw1 status + 0 0000000000000004 0000000000000000 0000000000000000 + 1 0000000200000025 0000000100059804 0000000000000000 + 2 0000000000000011 0000000000000000 0000000000000000 + + [...] + +What: /sys/kernel/debug/iommu/intel/dmar_perf_latency +Date: December 2023 +Contact: Jingqi Liu +Description: + This file is used to control and show counts of + execution time ranges for various types per DMAR. + + Firstly, write a value to + /sys/kernel/debug/iommu/intel/dmar_perf_latency + to enable sampling. + + The possible values are as follows: + + * 0 - disable sampling all latency data + + * 1 - enable sampling IOTLB invalidation latency data + + * 2 - enable sampling devTLB invalidation latency data + + * 3 - enable sampling intr entry cache invalidation latency data + + Next, read /sys/kernel/debug/iommu/intel/dmar_perf_latency gives + a snapshot of sampling result of all enabled monitors. + + Examples in Kabylake: + + :: + + 1) Disable sampling all latency data: + + $ sudo echo 0 > /sys/kernel/debug/iommu/intel/dmar_perf_latency + + 2) Enable sampling IOTLB invalidation latency data + + $ sudo echo 1 > /sys/kernel/debug/iommu/intel/dmar_perf_latency + + $ sudo cat /sys/kernel/debug/iommu/intel/dmar_perf_latency + + IOMMU: dmar0 Register Base Address: 26be37000 + <0.1us 0.1us-1us 1us-10us 10us-100us 100us-1ms + inv_iotlb 0 0 0 0 0 + + 1ms-10ms >=10ms min(us) max(us) average(us) + inv_iotlb 0 0 0 0 0 + + [...] + + IOMMU: dmar2 Register Base Address: fed91000 + <0.1us 0.1us-1us 1us-10us 10us-100us 100us-1ms + inv_iotlb 0 0 18 0 0 + + 1ms-10ms >=10ms min(us) max(us) average(us) + inv_iotlb 0 0 2 2 2 + + 3) Enable sampling devTLB invalidation latency data + + $ sudo echo 2 > /sys/kernel/debug/iommu/intel/dmar_perf_latency + + $ sudo cat /sys/kernel/debug/iommu/intel/dmar_perf_latency + + IOMMU: dmar0 Register Base Address: 26be37000 + <0.1us 0.1us-1us 1us-10us 10us-100us 100us-1ms + inv_devtlb 0 0 0 0 0 + + >=10ms min(us) max(us) average(us) + inv_devtlb 0 0 0 0 + + [...] + +What: /sys/kernel/debug/iommu/intel//domain_translation_struct +Date: December 2023 +Contact: Jingqi Liu +Description: + This file dumps a specified page table of Intel IOMMU + in legacy mode or scalable mode. + + For a device that only supports legacy mode, dump its + page table by the debugfs file in the debugfs device + directory. e.g. + /sys/kernel/debug/iommu/intel/0000:00:02.0/domain_translation_struct. + + For a device that supports scalable mode, dump the + page table of specified pasid by the debugfs file in + the debugfs pasid directory. e.g. + /sys/kernel/debug/iommu/intel/0000:00:02.0/1/domain_translation_struct. + + Examples in Kabylake: + + :: + + 1) Dump the page table of device "0000:00:02.0" that only supports legacy mode. + + $ sudo cat /sys/kernel/debug/iommu/intel/0000:00:02.0/domain_translation_struct + + Device 0000:00:02.0 @0x1017f8000 + IOVA_PFN PML5E PML4E + 0x000000008d800 | 0x0000000000000000 0x00000001017f9003 + 0x000000008d801 | 0x0000000000000000 0x00000001017f9003 + 0x000000008d802 | 0x0000000000000000 0x00000001017f9003 + + PDPE PDE PTE + 0x00000001017fa003 0x00000001017fb003 0x000000008d800003 + 0x00000001017fa003 0x00000001017fb003 0x000000008d801003 + 0x00000001017fa003 0x00000001017fb003 0x000000008d802003 + + [...] + + 2) Dump the page table of device "0000:00:0a.0" with PASID "1" that + supports scalable mode. + + $ sudo cat /sys/kernel/debug/iommu/intel/0000:00:0a.0/1/domain_translation_struct + + Device 0000:00:0a.0 with pasid 1 @0x10c112000 + IOVA_PFN PML5E PML4E + 0x0000000000000 | 0x0000000000000000 0x000000010df93003 + 0x0000000000001 | 0x0000000000000000 0x000000010df93003 + 0x0000000000002 | 0x0000000000000000 0x000000010df93003 + + PDPE PDE PTE + 0x0000000106ae6003 0x0000000104b38003 0x0000000147c00803 + 0x0000000106ae6003 0x0000000104b38003 0x0000000147c01803 + 0x0000000106ae6003 0x0000000104b38003 0x0000000147c02803 + + [...] diff --git a/Documentation/ABI/testing/gpio-cdev b/Documentation/ABI/testing/gpio-cdev index 66bdcd188b6c..c9689b2a6fed 100644 --- a/Documentation/ABI/testing/gpio-cdev +++ b/Documentation/ABI/testing/gpio-cdev @@ -6,8 +6,9 @@ Description: The character device files /dev/gpiochip* are the interface between GPIO chips and userspace. - The ioctl(2)-based ABI is defined and documented in - [include/uapi]. + The ioctl(2)-based ABI is defined in + [include/uapi] and documented in + Documentation/userspace-api/gpio/chardev.rst. The following file operations are supported: @@ -17,8 +18,8 @@ Description: ioctl(2) Initiate various actions. - See the inline documentation in [include/uapi] - for descriptions of all ioctls. + See Documentation/userspace-api/gpio/chardev.rst + for a description of all ioctls. close(2) Stops and free up the I/O contexts that was associated diff --git a/Documentation/ABI/testing/sysfs-class-hwmon b/Documentation/ABI/testing/sysfs-class-hwmon index 3dac923c9b0e..cfd0d0bab483 100644 --- a/Documentation/ABI/testing/sysfs-class-hwmon +++ b/Documentation/ABI/testing/sysfs-class-hwmon @@ -149,6 +149,15 @@ Description: RW +What: /sys/class/hwmon/hwmonX/inY_fault +Description: + Reports a voltage hard failure (eg: shorted component) + + - 1: Failed + - 0: Ok + + RO + What: /sys/class/hwmon/hwmonX/cpuY_vid Description: CPU core reference voltage. @@ -968,6 +977,15 @@ Description: RW +What: /sys/class/hwmon/hwmonX/humidityY_max_alarm +Description: + Maximum humidity detection + + - 0: OK + - 1: Maximum humidity detected + + RO + What: /sys/class/hwmon/hwmonX/humidityY_max_hyst Description: Humidity hysteresis value for max limit. @@ -987,6 +1005,15 @@ Description: RW +What: /sys/class/hwmon/hwmonX/humidityY_min_alarm +Description: + Minimum humidity detection + + - 0: OK + - 1: Minimum humidity detected + + RO + What: /sys/class/hwmon/hwmonX/humidityY_min_hyst Description: Humidity hysteresis value for min limit. diff --git a/Documentation/admin-guide/device-mapper/index.rst b/Documentation/admin-guide/device-mapper/index.rst index cde52cc09645..cc5aec861576 100644 --- a/Documentation/admin-guide/device-mapper/index.rst +++ b/Documentation/admin-guide/device-mapper/index.rst @@ -34,6 +34,8 @@ Device Mapper switch thin-provisioning unstriped + vdo-design + vdo verity writecache zero diff --git a/Documentation/admin-guide/device-mapper/vdo-design.rst b/Documentation/admin-guide/device-mapper/vdo-design.rst new file mode 100644 index 000000000000..3cd59decbec0 --- /dev/null +++ b/Documentation/admin-guide/device-mapper/vdo-design.rst @@ -0,0 +1,633 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +================ +Design of dm-vdo +================ + +The dm-vdo (virtual data optimizer) target provides inline deduplication, +compression, zero-block elimination, and thin provisioning. A dm-vdo target +can be backed by up to 256TB of storage, and can present a logical size of +up to 4PB. This target was originally developed at Permabit Technology +Corp. starting in 2009. It was first released in 2013 and has been used in +production environments ever since. It was made open-source in 2017 after +Permabit was acquired by Red Hat. This document describes the design of +dm-vdo. For usage, see vdo.rst in the same directory as this file. + +Because deduplication rates fall drastically as the block size increases, a +vdo target has a maximum block size of 4K. However, it can achieve +deduplication rates of 254:1, i.e. up to 254 copies of a given 4K block can +reference a single 4K of actual storage. It can achieve compression rates +of 14:1. All zero blocks consume no storage at all. + +Theory of Operation +=================== + +The design of dm-vdo is based on the idea that deduplication is a two-part +problem. The first is to recognize duplicate data. The second is to avoid +storing multiple copies of those duplicates. Therefore, dm-vdo has two main +parts: a deduplication index (called UDS) that is used to discover +duplicate data, and a data store with a reference counted block map that +maps from logical block addresses to the actual storage location of the +data. + +Zones and Threading +------------------- + +Due to the complexity of data optimization, the number of metadata +structures involved in a single write operation to a vdo target is larger +than most other targets. Furthermore, because vdo must operate on small +block sizes in order to achieve good deduplication rates, acceptable +performance can only be achieved through parallelism. Therefore, vdo's +design attempts to be lock-free. + +Most of a vdo's main data structures are designed to be easily divided into +"zones" such that any given bio must only access a single zone of any zoned +structure. Safety with minimal locking is achieved by ensuring that during +normal operation, each zone is assigned to a specific thread, and only that +thread will access the portion of the data structure in that zone. +Associated with each thread is a work queue. Each bio is associated with a +request object (the "data_vio") which will be added to a work queue when +the next phase of its operation requires access to the structures in the +zone associated with that queue. + +Another way of thinking about this arrangement is that the work queue for +each zone has an implicit lock on the structures it manages for all its +operations, because vdo guarantees that no other thread will alter those +structures. + +Although each structure is divided into zones, this division is not +reflected in the on-disk representation of each data structure. Therefore, +the number of zones for each structure, and hence the number of threads, +can be reconfigured each time a vdo target is started. + +The Deduplication Index +----------------------- + +In order to identify duplicate data efficiently, vdo was designed to +leverage some common characteristics of duplicate data. From empirical +observations, we gathered two key insights. The first is that in most data +sets with significant amounts of duplicate data, the duplicates tend to +have temporal locality. When a duplicate appears, it is more likely that +other duplicates will be detected, and that those duplicates will have been +written at about the same time. This is why the index keeps records in +temporal order. The second insight is that new data is more likely to +duplicate recent data than it is to duplicate older data and in general, +there are diminishing returns to looking further back in time. Therefore, +when the index is full, it should cull its oldest records to make space for +new ones. Another important idea behind the design of the index is that the +ultimate goal of deduplication is to reduce storage costs. Since there is a +trade-off between the storage saved and the resources expended to achieve +those savings, vdo does not attempt to find every last duplicate block. It +is sufficient to find and eliminate most of the redundancy. + +Each block of data is hashed to produce a 16-byte block name. An index +record consists of this block name paired with the presumed location of +that data on the underlying storage. However, it is not possible to +guarantee that the index is accurate. In the most common case, this occurs +because it is too costly to update the index when a block is over-written +or discarded. Doing so would require either storing the block name along +with the blocks, which is difficult to do efficiently in block-based +storage, or reading and rehashing each block before overwriting it. +Inaccuracy can also result from a hash collision where two different blocks +have the same name. In practice, this is extremely unlikely, but because +vdo does not use a cryptographic hash, a malicious workload could be +constructed. Because of these inaccuracies, vdo treats the locations in the +index as hints, and reads each indicated block to verify that it is indeed +a duplicate before sharing the existing block with a new one. + +Records are collected into groups called chapters. New records are added to +the newest chapter, called the open chapter. This chapter is stored in a +format optimized for adding and modifying records, and the content of the +open chapter is not finalized until it runs out of space for new records. +When the open chapter fills up, it is closed and a new open chapter is +created to collect new records. + +Closing a chapter converts it to a different format which is optimized for +reading. The records are written to a series of record pages based on the +order in which they were received. This means that records with temporal +locality should be on a small number of pages, reducing the I/O required to +retrieve them. The chapter also compiles an index that indicates which +record page contains any given name. This index means that a request for a +name can determine exactly which record page may contain that record, +without having to load the entire chapter from storage. This index uses +only a subset of the block name as its key, so it cannot guarantee that an +index entry refers to the desired block name. It can only guarantee that if +there is a record for this name, it will be on the indicated page. Closed +chapters are read-only structures and their contents are never altered in +any way. + +Once enough records have been written to fill up all the available index +space, the oldest chapter is removed to make space for new chapters. Any +time a request finds a matching record in the index, that record is copied +into the open chapter. This ensures that useful block names remain available +in the index, while unreferenced block names are forgotten over time. + +In order to find records in older chapters, the index also maintains a +higher level structure called the volume index, which contains entries +mapping each block name to the chapter containing its newest record. This +mapping is updated as records for the block name are copied or updated, +ensuring that only the newest record for a given block name can be found. +An older record for a block name will no longer be found even though it has +not been deleted from its chapter. Like the chapter index, the volume index +uses only a subset of the block name as its key and can not definitively +say that a record exists for a name. It can only say which chapter would +contain the record if a record exists. The volume index is stored entirely +in memory and is saved to storage only when the vdo target is shut down. + +From the viewpoint of a request for a particular block name, it will first +look up the name in the volume index. This search will either indicate that +the name is new, or which chapter to search. If it returns a chapter, the +request looks up its name in the chapter index. This will indicate either +that the name is new, or which record page to search. Finally, if it is not +new, the request will look for its name in the indicated record page. +This process may require up to two page reads per request (one for the +chapter index page and one for the request page). However, recently +accessed pages are cached so that these page reads can be amortized across +many block name requests. + +The volume index and the chapter indexes are implemented using a +memory-efficient structure called a delta index. Instead of storing the +entire block name (the key) for each entry, the entries are sorted by name +and only the difference between adjacent keys (the delta) is stored. +Because we expect the hashes to be randomly distributed, the size of the +deltas follows an exponential distribution. Because of this distribution, +the deltas are expressed using a Huffman code to take up even less space. +The entire sorted list of keys is called a delta list. This structure +allows the index to use many fewer bytes per entry than a traditional hash +table, but it is slightly more expensive to look up entries, because a +request must read every entry in a delta list to add up the deltas in order +to find the record it needs. The delta index reduces this lookup cost by +splitting its key space into many sub-lists, each starting at a fixed key +value, so that each individual list is short. + +The default index size can hold 64 million records, corresponding to about +256GB of data. This means that the index can identify duplicate data if the +original data was written within the last 256GB of writes. This range is +called the deduplication window. If new writes duplicate data that is older +than that, the index will not be able to find it because the records of the +older data have been removed. This means that if an application writes a +200 GB file to a vdo target and then immediately writes it again, the two +copies will deduplicate perfectly. Doing the same with a 500 GB file will +result in no deduplication, because the beginning of the file will no +longer be in the index by the time the second write begins (assuming there +is no duplication within the file itself). + +If an application anticipates a data workload that will see useful +deduplication beyond the 256GB threshold, vdo can be configured to use a +larger index with a correspondingly larger deduplication window. (This +configuration can only be set when the target is created, not altered +later. It is important to consider the expected workload for a vdo target +before configuring it.) There are two ways to do this. + +One way is to increase the memory size of the index, which also increases +the amount of backing storage required. Doubling the size of the index will +double the length of the deduplication window at the expense of doubling +the storage size and the memory requirements. + +The other option is to enable sparse indexing. Sparse indexing increases +the deduplication window by a factor of 10, at the expense of also +increasing the storage size by a factor of 10. However with sparse +indexing, the memory requirements do not increase. The trade-off is +slightly more computation per request and a slight decrease in the amount +of deduplication detected. For most workloads with significant amounts of +duplicate data, sparse indexing will detect 97-99% of the deduplication +that a standard index will detect. + +The vio and data_vio Structures +------------------------------- + +A vio (short for Vdo I/O) is conceptually similar to a bio, with additional +fields and data to track vdo-specific information. A struct vio maintains a +pointer to a bio but also tracks other fields specific to the operation of +vdo. The vio is kept separate from its related bio because there are many +circumstances where vdo completes the bio but must continue to do work +related to deduplication or compression. + +Metadata reads and writes, and other writes that originate within vdo, use +a struct vio directly. Application reads and writes use a larger structure +called a data_vio to track information about their progress. A struct +data_vio contain a struct vio and also includes several other fields +related to deduplication and other vdo features. The data_vio is the +primary unit of application work in vdo. Each data_vio proceeds through a +set of steps to handle the application data, after which it is reset and +returned to a pool of data_vios for reuse. + +There is a fixed pool of 2048 data_vios. This number was chosen to bound +the amount of work that is required to recover from a crash. In addition, +benchmarks have indicated that increasing the size of the pool does not +significantly improve performance. + +The Data Store +-------------- + +The data store is implemented by three main data structures, all of which +work in concert to reduce or amortize metadata updates across as many data +writes as possible. + +*The Slab Depot* + +Most of the vdo volume belongs to the slab depot. The depot contains a +collection of slabs. The slabs can be up to 32GB, and are divided into +three sections. Most of a slab consists of a linear sequence of 4K blocks. +These blocks are used either to store data, or to hold portions of the +block map (see below). In addition to the data blocks, each slab has a set +of reference counters, using 1 byte for each data block. Finally each slab +has a journal. + +Reference updates are written to the slab journal. Slab journal blocks are +written out either when they are full, or when the recovery journal +requests they do so in order to allow the main recovery journal (see below) +to free up space. The slab journal is used both to ensure that the main +recovery journal can regularly free up space, and also to amortize the cost +of updating individual reference blocks. The reference counters are kept in +memory and are written out, a block at a time in oldest-dirtied-order, only +when there is a need to reclaim slab journal space. The write operations +are performed in the background as needed so they do not add latency to +particular I/O operations. + +Each slab is independent of every other. They are assigned to "physical +zones" in round-robin fashion. If there are P physical zones, then slab n +is assigned to zone n mod P. + +The slab depot maintains an additional small data structure, the "slab +summary," which is used to reduce the amount of work needed to come back +online after a crash. The slab summary maintains an entry for each slab +indicating whether or not the slab has ever been used, whether all of its +reference count updates have been persisted to storage, and approximately +how full it is. During recovery, each physical zone will attempt to recover +at least one slab, stopping whenever it has recovered a slab which has some +free blocks. Once each zone has some space, or has determined that none is +available, the target can resume normal operation in a degraded mode. Read +and write requests can be serviced, perhaps with degraded performance, +while the remainder of the dirty slabs are recovered. + +*The Block Map* + +The block map contains the logical to physical mapping. It can be thought +of as an array with one entry per logical address. Each entry is 5 bytes, +36 bits of which contain the physical block number which holds the data for +the given logical address. The other 4 bits are used to indicate the nature +of the mapping. Of the 16 possible states, one represents a logical address +which is unmapped (i.e. it has never been written, or has been discarded), +one represents an uncompressed block, and the other 14 states are used to +indicate that the mapped data is compressed, and which of the compression +slots in the compressed block contains the data for this logical address. + +In practice, the array of mapping entries is divided into "block map +pages," each of which fits in a single 4K block. Each block map page +consists of a header and 812 mapping entries. Each mapping page is actually +a leaf of a radix tree which consists of block map pages at each level. +There are 60 radix trees which are assigned to "logical zones" in round +robin fashion. (If there are L logical zones, tree n will belong to zone n +mod L.) At each level, the trees are interleaved, so logical addresses +0-811 belong to tree 0, logical addresses 812-1623 belong to tree 1, and so +on. The interleaving is maintained all the way up to the 60 root nodes. +Choosing 60 trees results in an evenly distributed number of trees per zone +for a large number of possible logical zone counts. The storage for the 60 +tree roots is allocated at format time. All other block map pages are +allocated out of the slabs as needed. This flexible allocation avoids the +need to pre-allocate space for the entire set of logical mappings and also +makes growing the logical size of a vdo relatively easy. + +In operation, the block map maintains two caches. It is prohibitive to keep +the entire leaf level of the trees in memory, so each logical zone +maintains its own cache of leaf pages. The size of this cache is +configurable at target start time. The second cache is allocated at start +time, and is large enough to hold all the non-leaf pages of the entire +block map. This cache is populated as pages are needed. + +*The Recovery Journal* + +The recovery journal is used to amortize updates across the block map and +slab depot. Each write request causes an entry to be made in the journal. +Entries are either "data remappings" or "block map remappings." For a data +remapping, the journal records the logical address affected and its old and +new physical mappings. For a block map remapping, the journal records the +block map page number and the physical block allocated for it. Block map +pages are never reclaimed or repurposed, so the old mapping is always 0. + +Each journal entry is an intent record summarizing the metadata updates +that are required for a data_vio. The recovery journal issues a flush +before each journal block write to ensure that the physical data for the +new block mappings in that block are stable on storage, and journal block +writes are all issued with the FUA bit set to ensure the recovery journal +entries themselves are stable. The journal entry and the data write it +represents must be stable on disk before the other metadata structures may +be updated to reflect the operation. These entries allow the vdo device to +reconstruct the logical to physical mappings after an unexpected +interruption such as a loss of power. + +*Write Path* + +All write I/O to vdo is asynchronous. Each bio will be acknowledged as soon +as vdo has done enough work to guarantee that it can complete the write +eventually. Generally, the data for acknowledged but unflushed write I/O +can be treated as though it is cached in memory. If an application +requires data to be stable on storage, it must issue a flush or write the +data with the FUA bit set like any other asynchronous I/O. Shutting down +the vdo target will also flush any remaining I/O. + +Application write bios follow the steps outlined below. + +1. A data_vio is obtained from the data_vio pool and associated with the + application bio. If there are no data_vios available, the incoming bio + will block until a data_vio is available. This provides back pressure + to the application. The data_vio pool is protected by a spin lock. + + The newly acquired data_vio is reset and the bio's data is copied into + the data_vio if it is a write and the data is not all zeroes. The data + must be copied because the application bio can be acknowledged before + the data_vio processing is complete, which means later processing steps + will no longer have access to the application bio. The application bio + may also be smaller than 4K, in which case the data_vio will have + already read the underlying block and the data is instead copied over + the relevant portion of the larger block. + +2. The data_vio places a claim (the "logical lock") on the logical address + of the bio. It is vital to prevent simultaneous modifications of the + same logical address, because deduplication involves sharing blocks. + This claim is implemented as an entry in a hashtable where the key is + the logical address and the value is a pointer to the data_vio + currently handling that address. + + If a data_vio looks in the hashtable and finds that another data_vio is + already operating on that logical address, it waits until the previous + operation finishes. It also sends a message to inform the current + lock holder that it is waiting. Most notably, a new data_vio waiting + for a logical lock will flush the previous lock holder out of the + compression packer (step 8d) rather than allowing it to continue + waiting to be packed. + + This stage requires the data_vio to get an implicit lock on the + appropriate logical zone to prevent concurrent modifications of the + hashtable. This implicit locking is handled by the zone divisions + described above. + +3. The data_vio traverses the block map tree to ensure that all the + necessary internal tree nodes have been allocated, by trying to find + the leaf page for its logical address. If any interior tree page is + missing, it is allocated at this time out of the same physical storage + pool used to store application data. + + a. If any page-node in the tree has not yet been allocated, it must be + allocated before the write can continue. This step requires the + data_vio to lock the page-node that needs to be allocated. This + lock, like the logical block lock in step 2, is a hashtable entry + that causes other data_vios to wait for the allocation process to + complete. + + The implicit logical zone lock is released while the allocation is + happening, in order to allow other operations in the same logical + zone to proceed. The details of allocation are the same as in + step 4. Once a new node has been allocated, that node is added to + the tree using a similar process to adding a new data block mapping. + The data_vio journals the intent to add the new node to the block + map tree (step 10), updates the reference count of the new block + (step 11), and reacquires the implicit logical zone lock to add the + new mapping to the parent tree node (step 12). Once the tree is + updated, the data_vio proceeds down the tree. Any other data_vios + waiting on this allocation also proceed. + + b. In the steady-state case, the block map tree nodes will already be + allocated, so the data_vio just traverses the tree until it finds + the required leaf node. The location of the mapping (the "block map + slot") is recorded in the data_vio so that later steps do not need + to traverse the tree again. The data_vio then releases the implicit + logical zone lock. + +4. If the block is a zero block, skip to step 9. Otherwise, an attempt is + made to allocate a free data block. This allocation ensures that the + data_vio can write its data somewhere even if deduplication and + compression are not possible. This stage gets an implicit lock on a + physical zone to search for free space within that zone. + + The data_vio will search each slab in a zone until it finds a free + block or decides there are none. If the first zone has no free space, + it will proceed to search the next physical zone by taking the implicit + lock for that zone and releasing the previous one until it finds a + free block or runs out of zones to search. The data_vio will acquire a + struct pbn_lock (the "physical block lock") on the free block. The + struct pbn_lock also has several fields to record the various kinds of + claims that data_vios can have on physical blocks. The pbn_lock is + added to a hashtable like the logical block locks in step 2. This + hashtable is also covered by the implicit physical zone lock. The + reference count of the free block is updated to prevent any other + data_vio from considering it free. The reference counters are a + sub-component of the slab and are thus also covered by the implicit + physical zone lock. + +5. If an allocation was obtained, the data_vio has all the resources it + needs to complete the write. The application bio can safely be + acknowledged at this point. The acknowledgment happens on a separate + thread to prevent the application callback from blocking other data_vio + operations. + + If an allocation could not be obtained, the data_vio continues to + attempt to deduplicate or compress the data, but the bio is not + acknowledged because the vdo device may be out of space. + +6. At this point vdo must determine where to store the application data. + The data_vio's data is hashed and the hash (the "record name") is + recorded in the data_vio. + +7. The data_vio reserves or joins a struct hash_lock, which manages all of + the data_vios currently writing the same data. Active hash locks are + tracked in a hashtable similar to the way logical block locks are + tracked in step 2. This hashtable is covered by the implicit lock on + the hash zone. + + If there is no existing hash lock for this data_vio's record_name, the + data_vio obtains a hash lock from the pool, adds it to the hashtable, + and sets itself as the new hash lock's "agent." The hash_lock pool is + also covered by the implicit hash zone lock. The hash lock agent will + do all the work to decide where the application data will be + written. If a hash lock for the data_vio's record_name already exists, + and the data_vio's data is the same as the agent's data, the new + data_vio will wait for the agent to complete its work and then share + its result. + + In the rare case that a hash lock exists for the data_vio's hash but + the data does not match the hash lock's agent, the data_vio skips to + step 8h and attempts to write its data directly. This can happen if two + different data blocks produce the same hash, for example. + +8. The hash lock agent attempts to deduplicate or compress its data with + the following steps. + + a. The agent initializes and sends its embedded deduplication request + (struct uds_request) to the deduplication index. This does not + require the data_vio to get any locks because the index components + manage their own locking. The data_vio waits until it either gets a + response from the index or times out. + + b. If the deduplication index returns advice, the data_vio attempts to + obtain a physical block lock on the indicated physical address, in + order to read the data and verify that it is the same as the + data_vio's data, and that it can accept more references. If the + physical address is already locked by another data_vio, the data at + that address may soon be overwritten so it is not safe to use the + address for deduplication. + + c. If the data matches and the physical block can add references, the + agent and any other data_vios waiting on it will record this + physical block as their new physical address and proceed to step 9 + to record their new mapping. If there are more data_vios in the hash + lock than there are references available, one of the remaining + data_vios becomes the new agent and continues to step 8d as if no + valid advice was returned. + + d. If no usable duplicate block was found, the agent first checks that + it has an allocated physical block (from step 3) that it can write + to. If the agent does not have an allocation, some other data_vio in + the hash lock that does have an allocation takes over as agent. If + none of the data_vios have an allocated physical block, these writes + are out of space, so they proceed to step 13 for cleanup. + + e. The agent attempts to compress its data. If the data does not + compress, the data_vio will continue to step 8h to write its data + directly. + + If the compressed size is small enough, the agent will release the + implicit hash zone lock and go to the packer (struct packer) where + it will be placed in a bin (struct packer_bin) along with other + data_vios. All compression operations require the implicit lock on + the packer zone. + + The packer can combine up to 14 compressed blocks in a single 4k + data block. Compression is only helpful if vdo can pack at least 2 + data_vios into a single data block. This means that a data_vio may + wait in the packer for an arbitrarily long time for other data_vios + to fill out the compressed block. There is a mechanism for vdo to + evict waiting data_vios when continuing to wait would cause + problems. Circumstances causing an eviction include an application + flush, device shutdown, or a subsequent data_vio trying to overwrite + the same logical block address. A data_vio may also be evicted from + the packer if it cannot be paired with any other compressed block + before more compressible blocks need to use its bin. An evicted + data_vio will proceed to step 8h to write its data directly. + + f. If the agent fills a packer bin, either because all 14 of its slots + are used or because it has no remaining space, it is written out + using the allocated physical block from one of its data_vios. Step + 8d has already ensured that an allocation is available. + + g. Each data_vio sets the compressed block as its new physical address. + The data_vio obtains an implicit lock on the physical zone and + acquires the struct pbn_lock for the compressed block, which is + modified to be a shared lock. Then it releases the implicit physical + zone lock and proceeds to step 8i. + + h. Any data_vio evicted from the packer will have an allocation from + step 3. It will write its data to that allocated physical block. + + i. After the data is written, if the data_vio is the agent of a hash + lock, it will reacquire the implicit hash zone lock and share its + physical address with as many other data_vios in the hash lock as + possible. Each data_vio will then proceed to step 9 to record its + new mapping. + + j. If the agent actually wrote new data (whether compressed or not), + the deduplication index is updated to reflect the location of the + new data. The agent then releases the implicit hash zone lock. + +9. The data_vio determines the previous mapping of the logical address. + There is a cache for block map leaf pages (the "block map cache"), + because there are usually too many block map leaf nodes to store + entirely in memory. If the desired leaf page is not in the cache, the + data_vio will reserve a slot in the cache and load the desired page + into it, possibly evicting an older cached page. The data_vio then + finds the current physical address for this logical address (the "old + physical mapping"), if any, and records it. This step requires a lock + on the block map cache structures, covered by the implicit logical zone + lock. + +10. The data_vio makes an entry in the recovery journal containing the + logical block address, the old physical mapping, and the new physical + mapping. Making this journal entry requires holding the implicit + recovery journal lock. The data_vio will wait in the journal until all + recovery blocks up to the one containing its entry have been written + and flushed to ensure the transaction is stable on storage. + +11. Once the recovery journal entry is stable, the data_vio makes two slab + journal entries: an increment entry for the new mapping, and a + decrement entry for the old mapping. These two operations each require + holding a lock on the affected physical slab, covered by its implicit + physical zone lock. For correctness during recovery, the slab journal + entries in any given slab journal must be in the same order as the + corresponding recovery journal entries. Therefore, if the two entries + are in different zones, they are made concurrently, and if they are in + the same zone, the increment is always made before the decrement in + order to avoid underflow. After each slab journal entry is made in + memory, the associated reference count is also updated in memory. + +12. Once both of the reference count updates are done, the data_vio + acquires the implicit logical zone lock and updates the + logical-to-physical mapping in the block map to point to the new + physical block. At this point the write operation is complete. + +13. If the data_vio has a hash lock, it acquires the implicit hash zone + lock and releases its hash lock to the pool. + + The data_vio then acquires the implicit physical zone lock and releases + the struct pbn_lock it holds for its allocated block. If it had an + allocation that it did not use, it also sets the reference count for + that block back to zero to free it for use by subsequent data_vios. + + The data_vio then acquires the implicit logical zone lock and releases + the logical block lock acquired in step 2. + + The application bio is then acknowledged if it has not previously been + acknowledged, and the data_vio is returned to the pool. + +*Read Path* + +An application read bio follows a much simpler set of steps. It does steps +1 and 2 in the write path to obtain a data_vio and lock its logical +address. If there is already a write data_vio in progress for that logical +address that is guaranteed to complete, the read data_vio will copy the +data from the write data_vio and return it. Otherwise, it will look up the +logical-to-physical mapping by traversing the block map tree as in step 3, +and then read and possibly decompress the indicated data at the indicated +physical block address. A read data_vio will not allocate block map tree +nodes if they are missing. If the interior block map nodes do not exist +yet, the logical block map address must still be unmapped and the read +data_vio will return all zeroes. A read data_vio handles cleanup and +acknowledgment as in step 13, although it only needs to release the logical +lock and return itself to the pool. + +*Small Writes* + +All storage within vdo is managed as 4KB blocks, but it can accept writes +as small as 512 bytes. Processing a write that is smaller than 4K requires +a read-modify-write operation that reads the relevant 4K block, copies the +new data over the approriate sectors of the block, and then launches a +write operation for the modified data block. The read and write stages of +this operation are nearly identical to the normal read and write +operations, and a single data_vio is used throughout this operation. + +*Recovery* + +When a vdo is restarted after a crash, it will attempt to recover from the +recovery journal. During the pre-resume phase of the next start, the +recovery journal is read. The increment portion of valid entries are played +into the block map. Next, valid entries are played, in order as required, +into the slab journals. Finally, each physical zone attempts to replay at +least one slab journal to reconstruct the reference counts of one slab. +Once each zone has some free space (or has determined that it has none), +the vdo comes back online, while the remainder of the slab journals are +used to reconstruct the rest of the reference counts in the background. + +*Read-only Rebuild* + +If a vdo encounters an unrecoverable error, it will enter read-only mode. +This mode indicates that some previously acknowledged data may have been +lost. The vdo may be instructed to rebuild as best it can in order to +return to a writable state. However, this is never done automatically due +to the possibility that data has been lost. During a read-only rebuild, the +block map is recovered from the recovery journal as before. However, the +reference counts are not rebuilt from the slab journals. Instead, the +reference counts are zeroed, the entire block map is traversed, and the +reference counts are updated from the block mappings. While this may lose +some data, it ensures that the block map and reference counts are +consistent with each other. This allows vdo to resume normal operation and +accept further writes. diff --git a/Documentation/admin-guide/device-mapper/vdo.rst b/Documentation/admin-guide/device-mapper/vdo.rst new file mode 100644 index 000000000000..7e1ecafdf91e --- /dev/null +++ b/Documentation/admin-guide/device-mapper/vdo.rst @@ -0,0 +1,406 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +dm-vdo +====== + +The dm-vdo (virtual data optimizer) device mapper target provides +block-level deduplication, compression, and thin provisioning. As a device +mapper target, it can add these features to the storage stack, compatible +with any file system. The vdo target does not protect against data +corruption, relying instead on integrity protection of the storage below +it. It is strongly recommended that lvm be used to manage vdo volumes. See +lvmvdo(7). + +Userspace component +=================== + +Formatting a vdo volume requires the use of the 'vdoformat' tool, available +at: + +https://github.com/dm-vdo/vdo/ + +In most cases, a vdo target will recover from a crash automatically the +next time it is started. In cases where it encountered an unrecoverable +error (either during normal operation or crash recovery) the target will +enter or come up in read-only mode. Because read-only mode is indicative of +data-loss, a positive action must be taken to bring vdo out of read-only +mode. The 'vdoforcerebuild' tool, available from the same repo, is used to +prepare a read-only vdo to exit read-only mode. After running this tool, +the vdo target will rebuild its metadata the next time it is +started. Although some data may be lost, the rebuilt vdo's metadata will be +internally consistent and the target will be writable again. + +The repo also contains additional userspace tools which can be used to +inspect a vdo target's on-disk metadata. Fortunately, these tools are +rarely needed except by dm-vdo developers. + +Metadata requirements +===================== + +Each vdo volume reserves 3GB of space for metadata, or more depending on +its configuration. It is helpful to check that the space saved by +deduplication and compression is not cancelled out by the metadata +requirements. An estimation of the space saved for a specific dataset can +be computed with the vdo estimator tool, which is available at: + +https://github.com/dm-vdo/vdoestimator/ + +Target interface +================ + +Table line +---------- + +:: + + vdo V4 + + [optional arguments] + + +Required parameters: + + offset: + The offset, in sectors, at which the vdo volume's logical + space begins. + + logical device size: + The size of the device which the vdo volume will service, + in sectors. Must match the current logical size of the vdo + volume. + + storage device: + The device holding the vdo volume's data and metadata. + + storage device size: + The size of the device holding the vdo volume, as a number + of 4096-byte blocks. Must match the current size of the vdo + volume. + + minimum I/O size: + The minimum I/O size for this vdo volume to accept, in + bytes. Valid values are 512 or 4096. The recommended value + is 4096. + + block map cache size: + The size of the block map cache, as a number of 4096-byte + blocks. The minimum and recommended value is 32768 blocks. + If the logical thread count is non-zero, the cache size + must be at least 4096 blocks per logical thread. + + block map era length: + The speed with which the block map cache writes out + modified block map pages. A smaller era length is likely to + reduce the amount of time spent rebuilding, at the cost of + increased block map writes during normal operation. The + maximum and recommended value is 16380; the minimum value + is 1. + +Optional parameters: +-------------------- +Some or all of these parameters may be specified as pairs. + +Thread related parameters: + +Different categories of work are assigned to separate thread groups, and +the number of threads in each group can be configured separately. + +If , , and are all set to 0, the work handled by +all three thread types will be handled by a single thread. If any of these +values are non-zero, all of them must be non-zero. + + ack: + The number of threads used to complete bios. Since + completing a bio calls an arbitrary completion function + outside the vdo volume, threads of this type allow the vdo + volume to continue processing requests even when bio + completion is slow. The default is 1. + + bio: + The number of threads used to issue bios to the underlying + storage. Threads of this type allow the vdo volume to + continue processing requests even when bio submission is + slow. The default is 4. + + bioRotationInterval: + The number of bios to enqueue on each bio thread before + switching to the next thread. The value must be greater + than 0 and not more than 1024; the default is 64. + + cpu: + The number of threads used to do CPU-intensive work, such + as hashing and compression. The default is 1. + + hash: + The number of threads used to manage data comparisons for + deduplication based on the hash value of data blocks. The + default is 0. + + logical: + The number of threads used to manage caching and locking + based on the logical address of incoming bios. The default + is 0; the maximum is 60. + + physical: + The number of threads used to manage administration of the + underlying storage device. At format time, a slab size for + the vdo is chosen; the vdo storage device must be large + enough to have at least 1 slab per physical thread. The + default is 0; the maximum is 16. + +Miscellaneous parameters: + + maxDiscard: + The maximum size of discard bio accepted, in 4096-byte + blocks. I/O requests to a vdo volume are normally split + into 4096-byte blocks, and processed up to 2048 at a time. + However, discard requests to a vdo volume can be + automatically split to a larger size, up to + 4096-byte blocks in a single bio, and are limited to 1500 + at a time. Increasing this value may provide better overall + performance, at the cost of increased latency for the + individual discard requests. The default and minimum is 1; + the maximum is UINT_MAX / 4096. + + deduplication: + Whether deduplication is enabled. The default is 'on'; the + acceptable values are 'on' and 'off'. + + compression: + Whether compression is enabled. The default is 'off'; the + acceptable values are 'on' and 'off'. + +Device modification +------------------- + +A modified table may be loaded into a running, non-suspended vdo volume. +The modifications will take effect when the device is next resumed. The +modifiable parameters are , , +, , and . + +If the logical device size or physical device size are changed, upon +successful resume vdo will store the new values and require them on future +startups. These two parameters may not be decreased. The logical device +size may not exceed 4 PB. The physical device size must increase by at +least 32832 4096-byte blocks if at all, and must not exceed the size of the +underlying storage device. Additionally, when formatting the vdo device, a +slab size is chosen: the physical device size may never increase above the +size which provides 8192 slabs, and each increase must be large enough to +add at least one new slab. + +Examples: + +Start a previously-formatted vdo volume with 1 GB logical space and 1 GB +physical space, storing to /dev/dm-1 which has more than 1 GB of space. + +:: + + dmsetup create vdo0 --table \ + "0 2097152 vdo V4 /dev/dm-1 262144 4096 32768 16380" + +Grow the logical size to 4 GB. + +:: + + dmsetup reload vdo0 --table \ + "0 8388608 vdo V4 /dev/dm-1 262144 4096 32768 16380" + dmsetup resume vdo0 + +Grow the physical size to 2 GB. + +:: + + dmsetup reload vdo0 --table \ + "0 8388608 vdo V4 /dev/dm-1 524288 4096 32768 16380" + dmsetup resume vdo0 + +Grow the physical size by 1 GB more and increase max discard sectors. + +:: + + dmsetup reload vdo0 --table \ + "0 10485760 vdo V4 /dev/dm-1 786432 4096 32768 16380 maxDiscard 8" + dmsetup resume vdo0 + +Stop the vdo volume. + +:: + + dmsetup remove vdo0 + +Start the vdo volume again. Note that the logical and physical device sizes +must still match, but other parameters can change. + +:: + + dmsetup create vdo1 --table \ + "0 10485760 vdo V4 /dev/dm-1 786432 512 65550 5000 hash 1 logical 3 physical 2" + +Messages +-------- +All vdo devices accept messages in the form: + +:: + dmsetup message 0 + +The messages are: + + stats: + Outputs the current view of the vdo statistics. Mostly used + by the vdostats userspace program to interpret the output + buffer. + + dump: + Dumps many internal structures to the system log. This is + not always safe to run, so it should only be used to debug + a hung vdo. Optional parameters to specify structures to + dump are: + + viopool: The pool of I/O requests incoming bios + pools: A synonym of 'viopool' + vdo: Most of the structures managing on-disk data + queues: Basic information about each vdo thread + threads: A synonym of 'queues' + default: Equivalent to 'queues vdo' + all: All of the above. + + dump-on-shutdown: + Perform a default dump next time vdo shuts down. + + +Status +------ + +:: + + + + + device: + The name of the vdo volume. + + operating mode: + The current operating mode of the vdo volume; values may be + 'normal', 'recovering' (the volume has detected an issue + with its metadata and is attempting to repair itself), and + 'read-only' (an error has occurred that forces the vdo + volume to only support read operations and not writes). + + in recovery: + Whether the vdo volume is currently in recovery mode; + values may be 'recovering' or '-' which indicates not + recovering. + + index state: + The current state of the deduplication index in the vdo + volume; values may be 'closed', 'closing', 'error', + 'offline', 'online', 'opening', and 'unknown'. + + compression state: + The current state of compression in the vdo volume; values + may be 'offline' and 'online'. + + used physical blocks: + The number of physical blocks in use by the vdo volume. + + total physical blocks: + The total number of physical blocks the vdo volume may use; + the difference between this value and the + is the number of blocks the vdo + volume has left before being full. + +Memory Requirements +=================== + +A vdo target requires a fixed 38 MB of RAM along with the following amounts +that scale with the target: + +- 1.15 MB of RAM for each 1 MB of configured block map cache size. The + block map cache requires a minimum of 150 MB. +- 1.6 MB of RAM for each 1 TB of logical space. +- 268 MB of RAM for each 1 TB of physical storage managed by the volume. + +The deduplication index requires additional memory which scales with the +size of the deduplication window. For dense indexes, the index requires 1 +GB of RAM per 1 TB of window. For sparse indexes, the index requires 1 GB +of RAM per 10 TB of window. The index configuration is set when the target +is formatted and may not be modified. + +Module Parameters +================= + +The vdo driver has a numeric parameter 'log_level' which controls the +verbosity of logging from the driver. The default setting is 6 +(LOGLEVEL_INFO and more severe messages). + +Run-time Usage +============== + +When using dm-vdo, it is important to be aware of the ways in which its +behavior differs from other storage targets. + +- There is no guarantee that over-writes of existing blocks will succeed. + Because the underlying storage may be multiply referenced, over-writing + an existing block generally requires a vdo to have a free block + available. + +- When blocks are no longer in use, sending a discard request for those + blocks lets the vdo release references for those blocks. If the vdo is + thinly provisioned, discarding unused blocks is essential to prevent the + target from running out of space. However, due to the sharing of + duplicate blocks, no discard request for any given logical block is + guaranteed to reclaim space. + +- Assuming the underlying storage properly implements flush requests, vdo + is resilient against crashes, however, unflushed writes may or may not + persist after a crash. + +- Each write to a vdo target entails a significant amount of processing. + However, much of the work is paralellizable. Therefore, vdo targets + achieve better throughput at higher I/O depths, and can support up 2048 + requests in parallel. + +Tuning +====== + +The vdo device has many options, and it can be difficult to make optimal +choices without perfect knowledge of the workload. Additionally, most +configuration options must be set when a vdo target is started, and cannot +be changed without shutting it down completely; the configuration cannot be +changed while the target is active. Ideally, tuning with simulated +workloads should be performed before deploying vdo in production +environments. + +The most important value to adjust is the block map cache size. In order to +service a request for any logical address, a vdo must load the portion of +the block map which holds the relevant mapping. These mappings are cached. +Performance will suffer when the working set does not fit in the cache. By +default, a vdo allocates 128 MB of metadata cache in RAM to support +efficient access to 100 GB of logical space at a time. It should be scaled +up proportionally for larger working sets. + +The logical and physical thread counts should also be adjusted. A logical +thread controls a disjoint section of the block map, so additional logical +threads increase parallelism and can increase throughput. Physical threads +control a disjoint section of the data blocks, so additional physical +threads can also increase throughput. However, excess threads can waste +resources and increase contention. + +Bio submission threads control the parallelism involved in sending I/O to +the underlying storage; fewer threads mean there is more opportunity to +reorder I/O requests for performance benefit, but also that each I/O +request has to wait longer before being submitted. + +Bio acknowledgment threads are used for finishing I/O requests. This is +done on dedicated threads since the amount of work required to execute a +bio's callback can not be controlled by the vdo itself. Usually one thread +is sufficient but additional threads may be beneficial, particularly when +bios have CPU-heavy callbacks. + +CPU threads are used for hashing and for compression; in workloads with +compression enabled, more threads may result in higher throughput. + +Hash threads are used to sort active requests by hash and determine whether +they should deduplicate; the most CPU intensive actions done by these +threads are comparison of 4096-byte data blocks. In most cases, a single +hash thread is sufficient. diff --git a/Documentation/admin-guide/gpio/gpio-mockup.rst b/Documentation/admin-guide/gpio/gpio-mockup.rst index 493071da1738..d6e7438a7550 100644 --- a/Documentation/admin-guide/gpio/gpio-mockup.rst +++ b/Documentation/admin-guide/gpio/gpio-mockup.rst @@ -3,6 +3,14 @@ GPIO Testing Driver =================== +.. note:: + + This module has been obsoleted by the more flexible gpio-sim.rst. + New developments should use that API and existing developments are + encouraged to migrate as soon as possible. + This module will continue to be maintained but no new features will be + added. + The GPIO Testing Driver (gpio-mockup) provides a way to create simulated GPIO chips for testing purposes. The lines exposed by these chips can be accessed using the standard GPIO character device interface as well as manipulated diff --git a/Documentation/admin-guide/gpio/index.rst b/Documentation/admin-guide/gpio/index.rst index f6861ca16ffe..460afd29617e 100644 --- a/Documentation/admin-guide/gpio/index.rst +++ b/Documentation/admin-guide/gpio/index.rst @@ -1,16 +1,16 @@ .. SPDX-License-Identifier: GPL-2.0 ==== -gpio +GPIO ==== .. toctree:: :maxdepth: 1 + Character Device Userspace API <../../userspace-api/gpio/chardev> gpio-aggregator - sysfs - gpio-mockup gpio-sim + Obsolete APIs .. only:: subproject and html diff --git a/Documentation/admin-guide/gpio/obsolete.rst b/Documentation/admin-guide/gpio/obsolete.rst new file mode 100644 index 000000000000..5adbff02d61f --- /dev/null +++ b/Documentation/admin-guide/gpio/obsolete.rst @@ -0,0 +1,13 @@ +.. SPDX-License-Identifier: GPL-2.0 + +================== +Obsolete GPIO APIs +================== + +.. toctree:: + :maxdepth: 1 + + Character Device Userspace API (v1) <../../userspace-api/gpio/chardev_v1> + Sysfs Interface <../../userspace-api/gpio/sysfs> + Mockup Testing Module + diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 9fdbceb57247..0a589c46b59a 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -374,6 +374,11 @@ selects a performance level in this range and appropriate to the current workload. + amd_prefcore= + [X86] + disable + Disable amd-pstate preferred core. + amijoy.map= [HW,JOY] Amiga joystick support Map of devices attached to JOY0DAT and JOY1DAT Format: , @@ -1760,6 +1765,17 @@ (that will set all pages holding image data during restoration read-only). + hibernate.compressor= [HIBERNATION] Compression algorithm to be + used with hibernation. + Format: { lzo | lz4 } + Default: lzo + + lzo: Select LZO compression algorithm to + compress/decompress hibernation image. + + lz4: Select LZ4 compression algorithm to + compress/decompress hibernation image. + highmem=nn[KMG] [KNL,BOOT,EARLY] forces the highmem zone to have an exact size of . This works even on boxes that have no highmem otherwise. This also works to reduce highmem @@ -3775,10 +3791,6 @@ no5lvl [X86-64,RISCV,EARLY] Disable 5-level paging mode. Forces kernel to use 4-level paging instead. - noaliencache [MM, NUMA, SLAB] Disables the allocation of alien - caches in the slab allocator. Saves per-node memory, - but will impact performance. - noalign [KNL,ARM] noaltinstr [S390,EARLY] Disables alternative instructions @@ -5949,11 +5961,42 @@ simeth= [IA-64] simscsi= - slram= [HW,MTD] + slab_debug[=options[,slabs][;[options[,slabs]]...] [MM] + Enabling slab_debug allows one to determine the + culprit if slab objects become corrupted. Enabling + slab_debug can create guard zones around objects and + may poison objects when not in use. Also tracks the + last alloc / free. For more information see + Documentation/mm/slub.rst. + (slub_debug legacy name also accepted for now) + + slab_max_order= [MM] + Determines the maximum allowed order for slabs. + A high setting may cause OOMs due to memory + fragmentation. For more information see + Documentation/mm/slub.rst. + (slub_max_order legacy name also accepted for now) slab_merge [MM] Enable merging of slabs with similar size when the kernel is built without CONFIG_SLAB_MERGE_DEFAULT. + (slub_merge legacy name also accepted for now) + + slab_min_objects= [MM] + The minimum number of objects per slab. SLUB will + increase the slab order up to slab_max_order to + generate a sufficiently large slab able to contain + the number of objects indicated. The higher the number + of objects the smaller the overhead of tracking slabs + and the less frequently locks need to be acquired. + For more information see Documentation/mm/slub.rst. + (slub_min_objects legacy name also accepted for now) + + slab_min_order= [MM] + Determines the minimum page order for slabs. Must be + lower or equal to slab_max_order. For more information see + Documentation/mm/slub.rst. + (slub_min_order legacy name also accepted for now) slab_nomerge [MM] Disable merging of slabs with similar size. May be @@ -5967,47 +6010,9 @@ unchanged). Debug options disable merging on their own. For more information see Documentation/mm/slub.rst. + (slub_nomerge legacy name also accepted for now) - slab_max_order= [MM, SLAB] - Determines the maximum allowed order for slabs. - A high setting may cause OOMs due to memory - fragmentation. Defaults to 1 for systems with - more than 32MB of RAM, 0 otherwise. - - slub_debug[=options[,slabs][;[options[,slabs]]...] [MM, SLUB] - Enabling slub_debug allows one to determine the - culprit if slab objects become corrupted. Enabling - slub_debug can create guard zones around objects and - may poison objects when not in use. Also tracks the - last alloc / free. For more information see - Documentation/mm/slub.rst. - - slub_max_order= [MM, SLUB] - Determines the maximum allowed order for slabs. - A high setting may cause OOMs due to memory - fragmentation. For more information see - Documentation/mm/slub.rst. - - slub_min_objects= [MM, SLUB] - The minimum number of objects per slab. SLUB will - increase the slab order up to slub_max_order to - generate a sufficiently large slab able to contain - the number of objects indicated. The higher the number - of objects the smaller the overhead of tracking slabs - and the less frequently locks need to be acquired. - For more information see Documentation/mm/slub.rst. - - slub_min_order= [MM, SLUB] - Determines the minimum page order for slabs. Must be - lower than slub_max_order. - For more information see Documentation/mm/slub.rst. - - slub_merge [MM, SLUB] - Same with slab_merge. - - slub_nomerge [MM, SLUB] - Same with slab_nomerge. This is supported for legacy. - See slab_nomerge for more information. + slram= [HW,MTD] smart2= [HW] Format: [,[,...,]] diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index 9eb26014d34b..1e0d101b020a 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -300,8 +300,8 @@ platforms. The AMD P-States mechanism is the more performance and energy efficiency frequency management method on AMD processors. -AMD Pstate Driver Operation Modes -================================= +``amd-pstate`` Driver Operation Modes +====================================== ``amd_pstate`` CPPC has 3 operation modes: autonomous (active) mode, non-autonomous (passive) mode and guided autonomous (guided) mode. @@ -353,6 +353,48 @@ is activated. In this mode, driver requests minimum and maximum performance level and the platform autonomously selects a performance level in this range and appropriate to the current workload. +``amd-pstate`` Preferred Core +================================= + +The core frequency is subjected to the process variation in semiconductors. +Not all cores are able to reach the maximum frequency respecting the +infrastructure limits. Consequently, AMD has redefined the concept of +maximum frequency of a part. This means that a fraction of cores can reach +maximum frequency. To find the best process scheduling policy for a given +scenario, OS needs to know the core ordering informed by the platform through +highest performance capability register of the CPPC interface. + +``amd-pstate`` preferred core enables the scheduler to prefer scheduling on +cores that can achieve a higher frequency with lower voltage. The preferred +core rankings can dynamically change based on the workload, platform conditions, +thermals and ageing. + +The priority metric will be initialized by the ``amd-pstate`` driver. The ``amd-pstate`` +driver will also determine whether or not ``amd-pstate`` preferred core is +supported by the platform. + +``amd-pstate`` driver will provide an initial core ordering when the system boots. +The platform uses the CPPC interfaces to communicate the core ranking to the +operating system and scheduler to make sure that OS is choosing the cores +with highest performance firstly for scheduling the process. When ``amd-pstate`` +driver receives a message with the highest performance change, it will +update the core ranking and set the cpu's priority. + +``amd-pstate`` Preferred Core Switch +===================================== +Kernel Parameters +----------------- + +``amd-pstate`` peferred core`` has two states: enable and disable. +Enable/disable states can be chosen by different kernel parameters. +Default enable ``amd-pstate`` preferred core. + +``amd_prefcore=disable`` + +For systems that support ``amd-pstate`` preferred core, the core rankings will +always be advertised by the platform. But OS can choose to ignore that via the +kernel parameter ``amd_prefcore=disable``. + User Space Interface in ``sysfs`` - General =========================================== @@ -385,6 +427,19 @@ control its functionality at the system level. They are located in the to the operation mode represented by that string - or to be unregistered in the "disable" case. +``prefcore`` + Preferred core state of the driver: "enabled" or "disabled". + + "enabled" + Enable the ``amd-pstate`` preferred core. + + "disabled" + Disable the ``amd-pstate`` preferred core + + + This attribute is read-only to check the state of preferred core set + by the kernel parameter. + ``cpupower`` tool support for ``amd-pstate`` =============================================== diff --git a/Documentation/devicetree/bindings/ata/ahci-mtk.txt b/Documentation/devicetree/bindings/ata/ahci-mtk.txt deleted file mode 100644 index d2aa696b161b..000000000000 --- a/Documentation/devicetree/bindings/ata/ahci-mtk.txt +++ /dev/null @@ -1,51 +0,0 @@ -MediaTek Serial ATA controller - -Required properties: - - compatible : Must be "mediatek,-ahci", "mediatek,mtk-ahci". - When using "mediatek,mtk-ahci" compatible strings, you - need SoC specific ones in addition, one of: - - "mediatek,mt7622-ahci" - - reg : Physical base addresses and length of register sets. - - interrupts : Interrupt associated with the SATA device. - - interrupt-names : Associated name must be: "hostc". - - clocks : A list of phandle and clock specifier pairs, one for each - entry in clock-names. - - clock-names : Associated names must be: "ahb", "axi", "asic", "rbc", "pm". - - phys : A phandle and PHY specifier pair for the PHY port. - - phy-names : Associated name must be: "sata-phy". - - ports-implemented : See ./ahci-platform.txt for details. - -Optional properties: - - power-domains : A phandle and power domain specifier pair to the power - domain which is responsible for collapsing and restoring - power to the peripheral. - - resets : Must contain an entry for each entry in reset-names. - See ../reset/reset.txt for details. - - reset-names : Associated names must be: "axi", "sw", "reg". - - mediatek,phy-mode : A phandle to the system controller, used to enable - SATA function. - -Example: - - sata: sata@1a200000 { - compatible = "mediatek,mt7622-ahci", - "mediatek,mtk-ahci"; - reg = <0 0x1a200000 0 0x1100>; - interrupts = ; - interrupt-names = "hostc"; - clocks = <&pciesys CLK_SATA_AHB_EN>, - <&pciesys CLK_SATA_AXI_EN>, - <&pciesys CLK_SATA_ASIC_EN>, - <&pciesys CLK_SATA_RBC_EN>, - <&pciesys CLK_SATA_PM_EN>; - clock-names = "ahb", "axi", "asic", "rbc", "pm"; - phys = <&u3port1 PHY_TYPE_SATA>; - phy-names = "sata-phy"; - ports-implemented = <0x1>; - power-domains = <&scpsys MT7622_POWER_DOMAIN_HIF0>; - resets = <&pciesys MT7622_SATA_AXI_BUS_RST>, - <&pciesys MT7622_SATA_PHY_SW_RST>, - <&pciesys MT7622_SATA_PHY_REG_RST>; - reset-names = "axi", "sw", "reg"; - mediatek,phy-mode = <&pciesys>; - }; diff --git a/Documentation/devicetree/bindings/ata/atmel-at91_cf.txt b/Documentation/devicetree/bindings/ata/atmel-at91_cf.txt deleted file mode 100644 index c1d22b3ae134..000000000000 --- a/Documentation/devicetree/bindings/ata/atmel-at91_cf.txt +++ /dev/null @@ -1,19 +0,0 @@ -Atmel AT91RM9200 CompactFlash - -Required properties: -- compatible : "atmel,at91rm9200-cf". -- reg : should specify localbus address and size used. -- gpios : specifies the gpio pins to control the CF device. Detect - and reset gpio's are mandatory while irq and vcc gpio's are - optional and may be set to 0 if not present. - -Example: -compact-flash@50000000 { - compatible = "atmel,at91rm9200-cf"; - reg = <0x50000000 0x30000000>; - gpios = <&pioC 13 0 /* irq */ - &pioC 15 0 /* detect */ - 0 /* vcc */ - &pioC 5 0 /* reset */ - >; -}; diff --git a/Documentation/devicetree/bindings/ata/mediatek,mtk-ahci.yaml b/Documentation/devicetree/bindings/ata/mediatek,mtk-ahci.yaml new file mode 100644 index 000000000000..a34bd2e9c352 --- /dev/null +++ b/Documentation/devicetree/bindings/ata/mediatek,mtk-ahci.yaml @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/ata/mediatek,mtk-ahci.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek Serial ATA controller + +maintainers: + - Ryder Lee + +allOf: + - $ref: ahci-common.yaml# + +properties: + compatible: + items: + - enum: + - mediatek,mt7622-ahci + - const: mediatek,mtk-ahci + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + interrupt-names: + const: hostc + + clocks: + maxItems: 5 + + clock-names: + items: + - const: ahb + - const: axi + - const: asic + - const: rbc + - const: pm + + power-domains: + maxItems: 1 + + resets: + maxItems: 3 + + reset-names: + items: + - const: axi + - const: sw + - const: reg + + mediatek,phy-mode: + description: System controller phandle, used to enable SATA function + $ref: /schemas/types.yaml#/definitions/phandle + +required: + - reg + - interrupts + - interrupt-names + - clocks + - clock-names + - phys + - phy-names + - ports-implemented + +unevaluatedProperties: false + +examples: + - | + #include + #include + #include + #include + #include + + sata@1a200000 { + compatible = "mediatek,mt7622-ahci", "mediatek,mtk-ahci"; + reg = <0x1a200000 0x1100>; + interrupts = ; + interrupt-names = "hostc"; + clocks = <&pciesys CLK_SATA_AHB_EN>, + <&pciesys CLK_SATA_AXI_EN>, + <&pciesys CLK_SATA_ASIC_EN>, + <&pciesys CLK_SATA_RBC_EN>, + <&pciesys CLK_SATA_PM_EN>; + clock-names = "ahb", "axi", "asic", "rbc", "pm"; + phys = <&u3port1 PHY_TYPE_SATA>; + phy-names = "sata-phy"; + ports-implemented = <0x1>; + power-domains = <&scpsys MT7622_POWER_DOMAIN_HIF0>; + resets = <&pciesys MT7622_SATA_AXI_BUS_RST>, + <&pciesys MT7622_SATA_PHY_SW_RST>, + <&pciesys MT7622_SATA_PHY_REG_RST>; + reset-names = "axi", "sw", "reg"; + mediatek,phy-mode = <&pciesys>; + }; diff --git a/Documentation/devicetree/bindings/gpio/aspeed,ast2400-gpio.yaml b/Documentation/devicetree/bindings/gpio/aspeed,ast2400-gpio.yaml new file mode 100644 index 000000000000..cf11aa7ec8c7 --- /dev/null +++ b/Documentation/devicetree/bindings/gpio/aspeed,ast2400-gpio.yaml @@ -0,0 +1,148 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/gpio/aspeed,ast2400-gpio.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Aspeed GPIO controller + +maintainers: + - Andrew Jeffery + +properties: + compatible: + enum: + - aspeed,ast2400-gpio + - aspeed,ast2500-gpio + - aspeed,ast2600-gpio + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + description: The clock to use for debounce timings + + gpio-controller: true + gpio-line-names: + minItems: 36 + maxItems: 232 + + gpio-ranges: true + + "#gpio-cells": + const: 2 + + interrupts: + maxItems: 1 + + interrupt-controller: true + + "#interrupt-cells": + const: 2 + + ngpios: + minimum: 36 + maximum: 232 + +required: + - compatible + - reg + - interrupts + - interrupt-controller + - "#interrupt-cells" + - gpio-controller + - "#gpio-cells" + +allOf: + - if: + properties: + compatible: + contains: + const: aspeed,ast2400-gpio + then: + properties: + gpio-line-names: + minItems: 220 + maxItems: 220 + ngpios: + const: 220 + - if: + properties: + compatible: + contains: + const: aspeed,ast2500-gpio + then: + properties: + gpio-line-names: + minItems: 232 + maxItems: 232 + ngpios: + const: 232 + - if: + properties: + compatible: + contains: + const: aspeed,ast2600-gpio + then: + properties: + gpio-line-names: + minItems: 36 + maxItems: 208 + ngpios: + enum: [ 36, 208 ] + required: + - ngpios + +additionalProperties: false + +examples: + - | + gpio@1e780000 { + compatible = "aspeed,ast2400-gpio"; + reg = <0x1e780000 0x1000>; + interrupts = <20>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-controller; + #gpio-cells = <2>; + }; + - | + gpio: gpio@1e780000 { + compatible = "aspeed,ast2500-gpio"; + reg = <0x1e780000 0x200>; + interrupts = <20>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl 0 0 232>; + }; + - | + #include + #include + #include + gpio0: gpio@1e780000 { + compatible = "aspeed,ast2600-gpio"; + reg = <0x1e780000 0x400>; + clocks = <&syscon ASPEED_CLK_APB2>; + interrupts = ; + interrupt-controller; + #interrupt-cells = <2>; + #gpio-cells = <2>; + gpio-controller; + gpio-ranges = <&pinctrl 0 0 208>; + ngpios = <208>; + }; + gpio1: gpio@1e780800 { + compatible = "aspeed,ast2600-gpio"; + reg = <0x1e780800 0x800>; + clocks = <&syscon ASPEED_CLK_APB1>; + interrupts = ; + interrupt-controller; + #interrupt-cells = <2>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl 0 208 36>; + ngpios = <36>; + }; diff --git a/Documentation/devicetree/bindings/gpio/gpio-aspeed.txt b/Documentation/devicetree/bindings/gpio/gpio-aspeed.txt deleted file mode 100644 index b2033fc3a71a..000000000000 --- a/Documentation/devicetree/bindings/gpio/gpio-aspeed.txt +++ /dev/null @@ -1,39 +0,0 @@ -Aspeed GPIO controller Device Tree Bindings -------------------------------------------- - -Required properties: -- compatible : Either "aspeed,ast2400-gpio", "aspeed,ast2500-gpio", - or "aspeed,ast2600-gpio". - -- #gpio-cells : Should be two - - First cell is the GPIO line number - - Second cell is used to specify optional - parameters (unused) - -- reg : Address and length of the register set for the device -- gpio-controller : Marks the device node as a GPIO controller. -- interrupts : Interrupt specifier (see interrupt bindings for - details) -- interrupt-controller : Mark the GPIO controller as an interrupt-controller - -Optional properties: - -- clocks : A phandle to the clock to use for debounce timings -- ngpios : Number of GPIOs controlled by this controller. Should be set - when there are multiple GPIO controllers on a SoC (ast2600). - -The gpio and interrupt properties are further described in their respective -bindings documentation: - -- Documentation/devicetree/bindings/gpio/gpio.txt -- Documentation/devicetree/bindings/interrupt-controller/interrupts.txt - - Example: - gpio@1e780000 { - #gpio-cells = <2>; - compatible = "aspeed,ast2400-gpio"; - gpio-controller; - interrupts = <20>; - reg = <0x1e780000 0x1000>; - interrupt-controller; - }; diff --git a/Documentation/devicetree/bindings/gpio/gpio-mvebu.yaml b/Documentation/devicetree/bindings/gpio/gpio-mvebu.yaml index f1bd1e6b2e1f..33d4e4716516 100644 --- a/Documentation/devicetree/bindings/gpio/gpio-mvebu.yaml +++ b/Documentation/devicetree/bindings/gpio/gpio-mvebu.yaml @@ -115,7 +115,7 @@ allOf: required: - reg -unevaluatedProperties: true +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/gpio/gpio-pca9570.yaml b/Documentation/devicetree/bindings/gpio/gpio-pca9570.yaml index 452f8972a965..6f73961001b7 100644 --- a/Documentation/devicetree/bindings/gpio/gpio-pca9570.yaml +++ b/Documentation/devicetree/bindings/gpio/gpio-pca9570.yaml @@ -28,6 +28,9 @@ properties: minItems: 4 maxItems: 8 + label: + description: A descriptive name for this device. + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml b/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml index aa424e2b95f8..cc7a950a6030 100644 --- a/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml +++ b/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml @@ -53,6 +53,7 @@ properties: - renesas,gpio-r8a779a0 # R-Car V3U - renesas,gpio-r8a779f0 # R-Car S4-8 - renesas,gpio-r8a779g0 # R-Car V4H + - renesas,gpio-r8a779h0 # R-Car V4M - const: renesas,rcar-gen4-gpio # R-Car Gen4 reg: diff --git a/Documentation/devicetree/bindings/hwmon/adi,adm1177.yaml b/Documentation/devicetree/bindings/hwmon/adi,adm1177.yaml index 2e45364d0543..be7e9e91a3a8 100644 --- a/Documentation/devicetree/bindings/hwmon/adi,adm1177.yaml +++ b/Documentation/devicetree/bindings/hwmon/adi,adm1177.yaml @@ -46,7 +46,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: hwmon-common.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/hwmon/adi,adm1275.yaml b/Documentation/devicetree/bindings/hwmon/adi,adm1275.yaml index ab87f51c5aef..b68061294964 100644 --- a/Documentation/devicetree/bindings/hwmon/adi,adm1275.yaml +++ b/Documentation/devicetree/bindings/hwmon/adi,adm1275.yaml @@ -33,10 +33,6 @@ properties: reg: maxItems: 1 - shunt-resistor-micro-ohms: - description: - Shunt resistor value in micro-Ohm. - adi,volt-curr-sample-average: description: | Number of samples to be used to report voltage and current values. @@ -50,6 +46,7 @@ properties: enum: [1, 2, 4, 8, 16, 32, 64, 128] allOf: + - $ref: hwmon-common.yaml# - if: properties: compatible: @@ -107,7 +104,7 @@ required: - compatible - reg -additionalProperties: false +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/hwmon/adi,ltc2945.yaml b/Documentation/devicetree/bindings/hwmon/adi,ltc2945.yaml index 5cb66e97e816..6401b0a9aff4 100644 --- a/Documentation/devicetree/bindings/hwmon/adi,ltc2945.yaml +++ b/Documentation/devicetree/bindings/hwmon/adi,ltc2945.yaml @@ -31,7 +31,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: hwmon-common.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/hwmon/adi,ltc4282.yaml b/Documentation/devicetree/bindings/hwmon/adi,ltc4282.yaml new file mode 100644 index 000000000000..4854b95a93e3 --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/adi,ltc4282.yaml @@ -0,0 +1,159 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/hwmon/adi,ltc4282.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Analog Devices LTC4282 I2C High Current Hot Swap Controller over I2C + +maintainers: + - Nuno Sa + +description: | + Analog Devices LTC4282 I2C High Current Hot Swap Controller over I2C. + + https://www.analog.com/media/en/technical-documentation/data-sheets/ltc4282.pdf + +properties: + compatible: + enum: + - adi,ltc4282 + + reg: + maxItems: 1 + + vdd-supply: true + + clocks: + maxItems: 1 + + '#clock-cells': + const: 0 + + adi,rsense-nano-ohms: + description: Value of the sense resistor. + + adi,vin-mode-microvolt: + description: + Selects operating range for the Undervoltage, Overvoltage and Foldback + pins. Also for the ADC. Should be set to the nominal input voltage. + enum: [3300000, 5000000, 12000000, 24000000] + default: 12000000 + + adi,fet-bad-timeout-ms: + description: + From the moment a FET bad conditions is present, this property selects the + wait time/timeout for a FET-bad fault to be signaled. Setting this to 0, + disables FET bad faults to be reported. + default: 255 + maximum: 255 + + adi,overvoltage-dividers: + description: | + Select which dividers to use for VDD Overvoltage detection. Note that + when the internal dividers are used the threshold is referenced to VDD. + The percentages in the datasheet are misleading since the actual values + to look for are in the "Absolute Maximum Ratings" table in the + "Comparator Inputs" section. In there there's a line for each of the 5%, + 10% and 15% settings with the actual min, typical and max tolerances. + $ref: /schemas/types.yaml#/definitions/string + enum: [external, vdd_5_percent, vdd_10_percent, vdd_15_percent] + default: external + + adi,undervoltage-dividers: + description: | + Select which dividers to use for VDD Overvoltage detection. Note that + when the internal dividers are used the threshold is referenced to VDD. + The percentages in the datasheet are misleading since the actual values + to look for are in the "Absolute Maximum Ratings" table in the + "Comparator Inputs" section. In there there's a line for each of the 5%, + 10% and 15% settings with the actual min, typical and max tolerances. + $ref: /schemas/types.yaml#/definitions/string + enum: [external, vdd_5_percent, vdd_10_percent, vdd_15_percent] + default: external + + adi,current-limit-sense-microvolt: + description: + The current limit sense voltage of the chip is adjustable between + 12.5mV and 34.4mV in 3.1mV steps. This effectively limits the current + on the load. + enum: [12500, 15625, 18750, 21875, 25000, 28125, 31250, 34375] + default: 25000 + + adi,overcurrent-retry: + description: + If set, enables the chip to auto-retry 256 timer cycles after an + Overcurrent fault. + type: boolean + + adi,overvoltage-retry-disable: + description: + If set, disables the chip to auto-retry 50ms after an Overvoltage fault. + It's enabled by default. + type: boolean + + adi,undervoltage-retry-disable: + description: + If set, disables the chip to auto-retry 50ms after an Undervoltage fault. + It's enabled by default. + type: boolean + + adi,fault-log-enable: + description: + If set, enables the FAULT_LOG and ADC_ALERT_LOG registers to be written + to the EEPROM when a fault bit transitions high and hence, will be + available after a power cycle (the chip loads the contents of + the EE_FAULT_LOG register - the one in EEPROM - into FAULT_LOG at boot). + type: boolean + + adi,gpio1-mode: + description: Defines the function of the Pin. It can indicate that power is + good (PULL the pin low when power is not good) or that power is bad (Go + into high-z when power is not good). + $ref: /schemas/types.yaml#/definitions/string + enum: [power_bad, power_good] + default: power_good + + adi,gpio2-mode: + description: Defines the function of the Pin. It can be set as the input for + the ADC or indicating that the MOSFET is in stress (dissipating power). + $ref: /schemas/types.yaml#/definitions/string + enum: [adc_input, stress_fet] + default: adc_input + + adi,gpio3-monitor-enable: + description: If set, gpio3 is set as input for the ADC instead of gpio2. + type: boolean + +allOf: + - if: + required: + - adi,gpio3-monitor-enable + then: + properties: + adi,gpio2-mode: + const: stress_fet + +required: + - compatible + - reg + - adi,rsense-nano-ohms + +additionalProperties: false + +examples: + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + + hwmon@50 { + compatible = "adi,ltc4282"; + reg = <0x50>; + adi,rsense-nano-ohms = <500>; + + adi,gpio1-mode = "power_good"; + adi,gpio2-mode = "adc_input"; + }; + }; +... diff --git a/Documentation/devicetree/bindings/hwmon/amphenol,chipcap2.yaml b/Documentation/devicetree/bindings/hwmon/amphenol,chipcap2.yaml new file mode 100644 index 000000000000..17351fdbefce --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/amphenol,chipcap2.yaml @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/hwmon/amphenol,chipcap2.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ChipCap 2 humidity and temperature iio sensor + +maintainers: + - Javier Carrasco + +description: | + Relative humidity and temperature sensor on I2C bus. + + Datasheets: + https://www.amphenol-sensors.com/en/telaire/humidity/527-humidity-sensors/3095-chipcap-2 + +properties: + compatible: + oneOf: + - const: amphenol,cc2d23 + - items: + - enum: + - amphenol,cc2d23s + - amphenol,cc2d25 + - amphenol,cc2d25s + - amphenol,cc2d33 + - amphenol,cc2d33s + - amphenol,cc2d35 + - amphenol,cc2d35s + - const: amphenol,cc2d23 + + reg: + maxItems: 1 + + interrupts: + items: + - description: measurement ready indicator + - description: low humidity alarm + - description: high humidity alarm + + interrupt-names: + items: + - const: ready + - const: low + - const: high + + vdd-supply: + description: + Dedicated, controllable supply-regulator to reset the device and + enter in command mode. + +required: + - compatible + - reg + - vdd-supply + +additionalProperties: false + +examples: + - | + #include + i2c { + #address-cells = <1>; + #size-cells = <0>; + + humidity@28 { + compatible = "amphenol,cc2d23s", "amphenol,cc2d23"; + reg = <0x28>; + interrupt-parent = <&gpio>; + interrupts = <4 IRQ_TYPE_EDGE_RISING>, + <5 IRQ_TYPE_EDGE_RISING>, + <6 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "ready", "low", "high"; + vdd-supply = <®_vdd>; + }; + }; diff --git a/Documentation/devicetree/bindings/hwmon/aspeed,g6-pwm-tach.yaml b/Documentation/devicetree/bindings/hwmon/aspeed,g6-pwm-tach.yaml new file mode 100644 index 000000000000..9e5ed901ae54 --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/aspeed,g6-pwm-tach.yaml @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (C) 2023 Aspeed, Inc. +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/hwmon/aspeed,g6-pwm-tach.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ASPEED G6 PWM and Fan Tach controller + +maintainers: + - Billy Tsai + +description: | + The ASPEED PWM controller can support up to 16 PWM outputs. + The ASPEED Fan Tacho controller can support up to 16 fan tach input. + They are independent hardware blocks, which are different from the + previous version of the ASPEED chip. + +properties: + compatible: + enum: + - aspeed,ast2600-pwm-tach + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + resets: + maxItems: 1 + + "#pwm-cells": + const: 3 + +patternProperties: + "^fan-[0-9]+$": + $ref: fan-common.yaml# + unevaluatedProperties: false + required: + - tach-ch + +required: + - reg + - clocks + - resets + - "#pwm-cells" + - compatible + +additionalProperties: false + +examples: + - | + #include + pwm_tach: pwm-tach-controller@1e610000 { + compatible = "aspeed,ast2600-pwm-tach"; + reg = <0x1e610000 0x100>; + clocks = <&syscon ASPEED_CLK_AHB>; + resets = <&syscon ASPEED_RESET_PWM>; + #pwm-cells = <3>; + + fan-0 { + tach-ch = /bits/ 8 <0x0>; + pwms = <&pwm_tach 0 40000 0>; + }; + + fan-1 { + tach-ch = /bits/ 8 <0x1 0x2>; + pwms = <&pwm_tach 1 40000 0>; + }; + }; diff --git a/Documentation/devicetree/bindings/hwmon/fan-common.yaml b/Documentation/devicetree/bindings/hwmon/fan-common.yaml new file mode 100644 index 000000000000..0fb738081699 --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/fan-common.yaml @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/hwmon/fan-common.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Common Fan Properties + +maintainers: + - Naresh Solanki + - Billy Tsai + +properties: + max-rpm: + description: + Max RPM supported by fan. + $ref: /schemas/types.yaml#/definitions/uint32 + maximum: 100000 + + min-rpm: + description: + Min RPM supported by fan. + $ref: /schemas/types.yaml#/definitions/uint32 + maximum: 1000 + + pulses-per-revolution: + description: + The number of pulse from fan sensor per revolution. + $ref: /schemas/types.yaml#/definitions/uint32 + maximum: 4 + + tach-div: + description: + Divisor for the tach sampling clock, which determines the sensitivity of the tach pin. + $ref: /schemas/types.yaml#/definitions/uint32 + + target-rpm: + description: + The default desired fan speed in RPM. + $ref: /schemas/types.yaml#/definitions/uint32 + + fan-driving-mode: + description: + Select the driving mode of the fan.(DC, PWM and so on) + $ref: /schemas/types.yaml#/definitions/string + enum: [ dc, pwm ] + + pwms: + description: + PWM provider. + maxItems: 1 + + "#cooling-cells": + const: 2 + + cooling-levels: + description: + The control value which correspond to thermal cooling states. + $ref: /schemas/types.yaml#/definitions/uint32-array + + tach-ch: + description: + The tach channel used for the fan. + $ref: /schemas/types.yaml#/definitions/uint8-array + + label: + description: + Optional fan label + + fan-supply: + description: + Power supply for fan. + + reg: + maxItems: 1 + +additionalProperties: true + +... diff --git a/Documentation/devicetree/bindings/hwmon/hwmon-common.yaml b/Documentation/devicetree/bindings/hwmon/hwmon-common.yaml new file mode 100644 index 000000000000..dc86b5c72cf2 --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/hwmon-common.yaml @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/hwmon/hwmon-common.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Hardware Monitoring Devices Common Properties + +maintainers: + - Guenter Roeck + +properties: + label: + description: A descriptive name for this device. + + shunt-resistor-micro-ohms: + description: The value of current sense resistor. + +additionalProperties: true diff --git a/Documentation/devicetree/bindings/hwmon/lltc,ltc4151.yaml b/Documentation/devicetree/bindings/hwmon/lltc,ltc4151.yaml index e62aff670478..8f0095bb7f6e 100644 --- a/Documentation/devicetree/bindings/hwmon/lltc,ltc4151.yaml +++ b/Documentation/devicetree/bindings/hwmon/lltc,ltc4151.yaml @@ -25,7 +25,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: hwmon-common.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/hwmon/lltc,ltc4286.yaml b/Documentation/devicetree/bindings/hwmon/lltc,ltc4286.yaml index 98ca163d3486..853df9fef6c8 100644 --- a/Documentation/devicetree/bindings/hwmon/lltc,ltc4286.yaml +++ b/Documentation/devicetree/bindings/hwmon/lltc,ltc4286.yaml @@ -25,15 +25,14 @@ properties: The default is 102.4 volts. type: boolean - shunt-resistor-micro-ohms: - description: - Resistor value micro-ohms. - required: - compatible - reg -additionalProperties: false +allOf: + - $ref: hwmon-common.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/hwmon/lm75.yaml b/Documentation/devicetree/bindings/hwmon/lm75.yaml index ed269e428a3d..29bd7460cc26 100644 --- a/Documentation/devicetree/bindings/hwmon/lm75.yaml +++ b/Documentation/devicetree/bindings/hwmon/lm75.yaml @@ -57,6 +57,7 @@ required: - reg allOf: + - $ref: hwmon-common.yaml# - if: not: properties: @@ -71,7 +72,7 @@ allOf: properties: interrupts: false -additionalProperties: false +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/hwmon/nuvoton,nct6775.yaml b/Documentation/devicetree/bindings/hwmon/nuvoton,nct6775.yaml index 358b262431fc..e3db642878d4 100644 --- a/Documentation/devicetree/bindings/hwmon/nuvoton,nct6775.yaml +++ b/Documentation/devicetree/bindings/hwmon/nuvoton,nct6775.yaml @@ -25,6 +25,7 @@ properties: - nuvoton,nct6796 - nuvoton,nct6797 - nuvoton,nct6798 + - nuvoton,nct6799 reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/hwmon/pmbus/infineon,tda38640.yaml b/Documentation/devicetree/bindings/hwmon/pmbus/infineon,tda38640.yaml index ded1c115764b..5c4e52b472ad 100644 --- a/Documentation/devicetree/bindings/hwmon/pmbus/infineon,tda38640.yaml +++ b/Documentation/devicetree/bindings/hwmon/pmbus/infineon,tda38640.yaml @@ -30,6 +30,23 @@ properties: unconnected(has internal pull-down). type: boolean + interrupts: + maxItems: 1 + + regulators: + type: object + description: + list of regulators provided by this controller. + + properties: + vout: + $ref: /schemas/regulator/regulator.yaml# + type: object + + unevaluatedProperties: false + + additionalProperties: false + required: - compatible - reg @@ -38,6 +55,7 @@ additionalProperties: false examples: - | + #include i2c { #address-cells = <1>; #size-cells = <0>; @@ -45,5 +63,15 @@ examples: tda38640@40 { compatible = "infineon,tda38640"; reg = <0x40>; + + interrupt-parent = <&smb_pex_cpu0_event>; + interrupts = <10 IRQ_TYPE_LEVEL_LOW>; + + regulators { + pvnn_main_cpu0: vout { + regulator-name = "pvnn_main_cpu0"; + regulator-enable-ramp-delay = <200>; + }; + }; }; }; diff --git a/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml b/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml index da8292bc32f5..a20f140dc79a 100644 --- a/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml +++ b/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml @@ -34,11 +34,26 @@ properties: Shunt (sense) resistor value in micro-Ohms default: 1000 + regulators: + type: object + + properties: + vout: + $ref: /schemas/regulator/regulator.yaml# + type: object + + unevaluatedProperties: false + + additionalProperties: false + required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/hwmon/hwmon-common.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml b/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml index 378d1f6aeeb3..df86c2c92037 100644 --- a/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml +++ b/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml @@ -28,10 +28,14 @@ properties: - ti,ina231 - ti,ina237 - ti,ina238 + - ti,ina260 reg: maxItems: 1 + "#io-channel-cells": + const: 1 + shunt-resistor: description: Shunt resistor value in micro-Ohm. @@ -66,7 +70,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: hwmon-common.yaml# + +unevaluatedProperties: false examples: - | @@ -77,6 +84,8 @@ examples: power-sensor@44 { compatible = "ti,ina220"; reg = <0x44>; + #io-channel-cells = <1>; + label = "vdd_3v0"; shunt-resistor = <1000>; vs-supply = <&vdd_3v0>; }; diff --git a/Documentation/devicetree/bindings/hwmon/ti,tmp513.yaml b/Documentation/devicetree/bindings/hwmon/ti,tmp513.yaml index cdd1489e0c54..227858e76058 100644 --- a/Documentation/devicetree/bindings/hwmon/ti,tmp513.yaml +++ b/Documentation/devicetree/bindings/hwmon/ti,tmp513.yaml @@ -72,7 +72,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: hwmon-common.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/hwmon/ti,tps23861.yaml b/Documentation/devicetree/bindings/hwmon/ti,tps23861.yaml index ebc8d466c1aa..f58248c29e22 100644 --- a/Documentation/devicetree/bindings/hwmon/ti,tps23861.yaml +++ b/Documentation/devicetree/bindings/hwmon/ti,tps23861.yaml @@ -35,7 +35,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: hwmon-common.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index a4042ae24770..5c130cf06a21 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -83,6 +83,7 @@ properties: - description: Qcom Adreno GPUs implementing "qcom,smmu-500" and "arm,mmu-500" items: - enum: + - qcom,qcm2290-smmu-500 - qcom,sa8775p-smmu-500 - qcom,sc7280-smmu-500 - qcom,sc8280xp-smmu-500 @@ -93,6 +94,7 @@ properties: - qcom,sm8350-smmu-500 - qcom,sm8450-smmu-500 - qcom,sm8550-smmu-500 + - qcom,sm8650-smmu-500 - const: qcom,adreno-smmu - const: qcom,smmu-500 - const: arm,mmu-500 @@ -462,6 +464,7 @@ allOf: compatible: items: - enum: + - qcom,qcm2290-smmu-500 - qcom,sm6115-smmu-500 - qcom,sm6125-smmu-500 - const: qcom,adreno-smmu @@ -484,7 +487,12 @@ allOf: - if: properties: compatible: - const: qcom,sm8450-smmu-500 + items: + - const: qcom,sm8450-smmu-500 + - const: qcom,adreno-smmu + - const: qcom,smmu-500 + - const: arm,mmu-500 + then: properties: clock-names: @@ -508,7 +516,13 @@ allOf: - if: properties: compatible: - const: qcom,sm8550-smmu-500 + items: + - enum: + - qcom,sm8550-smmu-500 + - qcom,sm8650-smmu-500 + - const: qcom,adreno-smmu + - const: qcom,smmu-500 + - const: arm,mmu-500 then: properties: clock-names: @@ -534,7 +548,6 @@ allOf: - cavium,smmu-v2 - marvell,ap806-smmu-500 - nvidia,smmu-500 - - qcom,qcm2290-smmu-500 - qcom,qdu1000-smmu-500 - qcom,sc7180-smmu-500 - qcom,sc8180x-smmu-500 @@ -544,7 +557,6 @@ allOf: - qcom,sdx65-smmu-500 - qcom,sm6350-smmu-500 - qcom,sm6375-smmu-500 - - qcom,sm8650-smmu-500 - qcom,x1e80100-smmu-500 then: properties: diff --git a/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml b/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml index 82eb7a24c857..82f7ee8702cb 100644 --- a/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml +++ b/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.yaml @@ -55,8 +55,9 @@ properties: - enum: - fsl,imx8mn-usdhc - fsl,imx8mp-usdhc - - fsl,imx93-usdhc - fsl,imx8ulp-usdhc + - fsl,imx93-usdhc + - fsl,imx95-usdhc - const: fsl,imx8mm-usdhc - items: - enum: @@ -162,6 +163,9 @@ properties: - const: ahb - const: per + iommus: + maxItems: 1 + power-domains: maxItems: 1 @@ -173,6 +177,11 @@ properties: - const: state_100mhz - const: state_200mhz - const: sleep + - minItems: 2 + items: + - const: default + - const: state_100mhz + - const: sleep - minItems: 1 items: - const: default diff --git a/Documentation/devicetree/bindings/mmc/fsl-imx-mmc.yaml b/Documentation/devicetree/bindings/mmc/fsl-imx-mmc.yaml index 221f5bc047bd..7911316fbd6a 100644 --- a/Documentation/devicetree/bindings/mmc/fsl-imx-mmc.yaml +++ b/Documentation/devicetree/bindings/mmc/fsl-imx-mmc.yaml @@ -24,6 +24,14 @@ properties: reg: maxItems: 1 + clocks: + maxItems: 2 + + clock-names: + items: + - const: ipg + - const: per + interrupts: maxItems: 1 @@ -34,6 +42,8 @@ properties: const: rx-tx required: + - clocks + - clock-names - compatible - reg - interrupts @@ -46,6 +56,8 @@ examples: compatible = "fsl,imx27-mmc", "fsl,imx21-mmc"; reg = <0x10014000 0x1000>; interrupts = <11>; + clocks = <&clks 29>, <&clks 60>; + clock-names = "ipg", "per"; dmas = <&dma 7>; dma-names = "rx-tx"; bus-width = <4>; diff --git a/Documentation/devicetree/bindings/mmc/hi3798cv200-dw-mshc.txt b/Documentation/devicetree/bindings/mmc/hi3798cv200-dw-mshc.txt deleted file mode 100644 index a0693b7145f2..000000000000 --- a/Documentation/devicetree/bindings/mmc/hi3798cv200-dw-mshc.txt +++ /dev/null @@ -1,40 +0,0 @@ -* Hisilicon Hi3798CV200 specific extensions to the Synopsys Designware Mobile - Storage Host Controller - -Read synopsys-dw-mshc.txt for more details - -The Synopsys designware mobile storage host controller is used to interface -a SoC with storage medium such as eMMC or SD/MMC cards. This file documents -differences between the core Synopsys dw mshc controller properties described -by synopsys-dw-mshc.txt and the properties used by the Hisilicon Hi3798CV200 -specific extensions to the Synopsys Designware Mobile Storage Host Controller. - -Required Properties: -- compatible: Should contain "hisilicon,hi3798cv200-dw-mshc". -- clocks: A list of phandle + clock-specifier pairs for the clocks listed - in clock-names. -- clock-names: Should contain the following: - "ciu" - The ciu clock described in synopsys-dw-mshc.txt. - "biu" - The biu clock described in synopsys-dw-mshc.txt. - "ciu-sample" - Hi3798CV200 extended phase clock for ciu sampling. - "ciu-drive" - Hi3798CV200 extended phase clock for ciu driving. - -Example: - - emmc: mmc@9830000 { - compatible = "hisilicon,hi3798cv200-dw-mshc"; - reg = <0x9830000 0x10000>; - interrupts = ; - clocks = <&crg HISTB_MMC_CIU_CLK>, - <&crg HISTB_MMC_BIU_CLK>, - <&crg HISTB_MMC_SAMPLE_CLK>, - <&crg HISTB_MMC_DRV_CLK>; - clock-names = "ciu", "biu", "ciu-sample", "ciu-drive"; - fifo-depth = <256>; - clock-frequency = <200000000>; - cap-mmc-highspeed; - mmc-ddr-1_8v; - mmc-hs200-1_8v; - non-removable; - bus-width = <8>; - }; diff --git a/Documentation/devicetree/bindings/mmc/hisilicon,hi3798cv200-dw-mshc.yaml b/Documentation/devicetree/bindings/mmc/hisilicon,hi3798cv200-dw-mshc.yaml new file mode 100644 index 000000000000..41c9b22523e7 --- /dev/null +++ b/Documentation/devicetree/bindings/mmc/hisilicon,hi3798cv200-dw-mshc.yaml @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mmc/hisilicon,hi3798cv200-dw-mshc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Hisilicon HiSTB SoCs specific extensions to the Synopsys DWMMC controller + +maintainers: + - Yang Xiwen + +properties: + compatible: + enum: + - hisilicon,hi3798cv200-dw-mshc + - hisilicon,hi3798mv200-dw-mshc + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + items: + - description: bus interface unit clock + - description: card interface unit clock + - description: card input sample phase clock + - description: controller output drive phase clock + + clock-names: + items: + - const: ciu + - const: biu + - const: ciu-sample + - const: ciu-drive + + hisilicon,sap-dll-reg: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: | + DWMMC core on Hi3798MV2x SoCs has a delay-locked-loop(DLL) attached to card data input path. + It is integrated into CRG core on the SoC and has to be controlled during tuning. + items: + - description: A phandle pointed to the CRG syscon node + - description: Sample DLL register offset in CRG address space + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + +allOf: + - $ref: synopsys-dw-mshc-common.yaml# + + - if: + properties: + compatible: + contains: + const: hisilicon,hi3798mv200-dw-mshc + then: + required: + - hisilicon,sap-dll-reg + else: + properties: + hisilicon,sap-dll-reg: false + +unevaluatedProperties: false + +examples: + - | + #include + #include + + mmc@9830000 { + compatible = "hisilicon,hi3798cv200-dw-mshc"; + reg = <0x9830000 0x10000>; + interrupts = ; + clocks = <&crg HISTB_MMC_CIU_CLK>, + <&crg HISTB_MMC_BIU_CLK>, + <&crg HISTB_MMC_SAMPLE_CLK>, + <&crg HISTB_MMC_DRV_CLK>; + clock-names = "ciu", "biu", "ciu-sample", "ciu-drive"; + resets = <&crg 0xa0 4>; + reset-names = "reset"; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pins_1 &emmc_pins_2 + &emmc_pins_3 &emmc_pins_4>; + fifo-depth = <256>; + clock-frequency = <200000000>; + cap-mmc-highspeed; + mmc-ddr-1_8v; + mmc-hs200-1_8v; + non-removable; + bus-width = <8>; + }; diff --git a/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml b/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml index f7a4c6bc70f6..29f2400247eb 100644 --- a/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml +++ b/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml @@ -67,6 +67,7 @@ properties: - renesas,sdhi-r8a779a0 # R-Car V3U - renesas,sdhi-r8a779f0 # R-Car S4-8 - renesas,sdhi-r8a779g0 # R-Car V4H + - renesas,sdhi-r8a779h0 # R-Car V4M - const: renesas,rcar-gen4-sdhi # R-Car Gen4 reg: diff --git a/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml b/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml index 42804d955293..4d3031d9965f 100644 --- a/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml +++ b/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml @@ -19,6 +19,8 @@ properties: - rockchip,rk3568-dwcmshc - rockchip,rk3588-dwcmshc - snps,dwcmshc-sdhci + - sophgo,cv1800b-dwcmshc + - sophgo,sg2002-dwcmshc - thead,th1520-dwcmshc reg: diff --git a/Documentation/devicetree/bindings/opp/opp-v2-base.yaml b/Documentation/devicetree/bindings/opp/opp-v2-base.yaml index e2f8f7af3cf4..b1bb87c865ed 100644 --- a/Documentation/devicetree/bindings/opp/opp-v2-base.yaml +++ b/Documentation/devicetree/bindings/opp/opp-v2-base.yaml @@ -57,8 +57,6 @@ patternProperties: specific binding. minItems: 1 maxItems: 32 - items: - maxItems: 1 opp-microvolt: description: | diff --git a/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml b/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml index 2ff246cf8b81..929b7ef9c1bc 100644 --- a/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml +++ b/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml @@ -24,6 +24,8 @@ properties: - qcom,msm8917-rpmpd - qcom,msm8939-rpmpd - qcom,msm8953-rpmpd + - qcom,msm8974-rpmpd + - qcom,msm8974pro-pma8084-rpmpd - qcom,msm8976-rpmpd - qcom,msm8994-rpmpd - qcom,msm8996-rpmpd diff --git a/Documentation/devicetree/bindings/pwm/atmel,hlcdc-pwm.yaml b/Documentation/devicetree/bindings/pwm/atmel,hlcdc-pwm.yaml new file mode 100644 index 000000000000..0e92868a2b68 --- /dev/null +++ b/Documentation/devicetree/bindings/pwm/atmel,hlcdc-pwm.yaml @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pwm/atmel,hlcdc-pwm.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Atmel's HLCDC's PWM controller + +maintainers: + - Nicolas Ferre + - Alexandre Belloni + - Claudiu Beznea + +description: + The LCDC integrates a Pulse Width Modulation (PWM) Controller. This block + generates the LCD contrast control signal (LCD_PWM) that controls the + display's contrast by software. LCDC_PWM is an 8-bit PWM signal that can be + converted to an analog voltage with a simple passive filter. LCD display + panels have different backlight specifications in terms of minimum/maximum + values for PWM frequency. If the LCDC PWM frequency range does not match the + LCD display panel, it is possible to use the standalone PWM Controller to + drive the backlight. + +properties: + compatible: + const: atmel,hlcdc-pwm + + "#pwm-cells": + const: 3 + +required: + - compatible + - "#pwm-cells" + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/pwm/atmel-hlcdc-pwm.txt b/Documentation/devicetree/bindings/pwm/atmel-hlcdc-pwm.txt deleted file mode 100644 index afa501bf7f94..000000000000 --- a/Documentation/devicetree/bindings/pwm/atmel-hlcdc-pwm.txt +++ /dev/null @@ -1,29 +0,0 @@ -Device-Tree bindings for Atmel's HLCDC (High-end LCD Controller) PWM driver - -The Atmel HLCDC PWM is subdevice of the HLCDC MFD device. -See ../mfd/atmel-hlcdc.txt for more details. - -Required properties: - - compatible: value should be one of the following: - "atmel,hlcdc-pwm" - - pinctr-names: the pin control state names. Should contain "default". - - pinctrl-0: should contain the pinctrl states described by pinctrl - default. - - #pwm-cells: should be set to 3. This PWM chip use the default 3 cells - bindings defined in pwm.yaml in this directory. - -Example: - - hlcdc: hlcdc@f0030000 { - compatible = "atmel,sama5d3-hlcdc"; - reg = <0xf0030000 0x2000>; - clocks = <&lcdc_clk>, <&lcdck>, <&clk32k>; - clock-names = "periph_clk","sys_clk", "slow_clk"; - - hlcdc_pwm: hlcdc-pwm { - compatible = "atmel,hlcdc-pwm"; - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_lcd_pwm>; - #pwm-cells = <3>; - }; - }; diff --git a/Documentation/devicetree/bindings/pwm/marvell,pxa-pwm.yaml b/Documentation/devicetree/bindings/pwm/marvell,pxa-pwm.yaml new file mode 100644 index 000000000000..ba6325575ea0 --- /dev/null +++ b/Documentation/devicetree/bindings/pwm/marvell,pxa-pwm.yaml @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pwm/marvell,pxa-pwm.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Marvell PXA PWM + +maintainers: + - Duje Mihanović + +allOf: + - $ref: pwm.yaml# + +properties: + compatible: + enum: + - marvell,pxa250-pwm + - marvell,pxa270-pwm + - marvell,pxa168-pwm + - marvell,pxa910-pwm + + reg: + # Length should be 0x10 + maxItems: 1 + + "#pwm-cells": + # Used for specifying the period length in nanoseconds + const: 1 + + clocks: + maxItems: 1 + +required: + - compatible + - reg + - "#pwm-cells" + - clocks + +additionalProperties: false + +examples: + - | + #include + + pwm0: pwm@40b00000 { + compatible = "marvell,pxa250-pwm"; + reg = <0x40b00000 0x10>; + #pwm-cells = <1>; + clocks = <&clks CLK_PWM0>; + }; diff --git a/Documentation/devicetree/bindings/pwm/mediatek,mt2712-pwm.yaml b/Documentation/devicetree/bindings/pwm/mediatek,mt2712-pwm.yaml index 0fbe8a6469eb..a5c308801619 100644 --- a/Documentation/devicetree/bindings/pwm/mediatek,mt2712-pwm.yaml +++ b/Documentation/devicetree/bindings/pwm/mediatek,mt2712-pwm.yaml @@ -24,6 +24,7 @@ properties: - mediatek,mt7629-pwm - mediatek,mt7981-pwm - mediatek,mt7986-pwm + - mediatek,mt7988-pwm - mediatek,mt8183-pwm - mediatek,mt8365-pwm - mediatek,mt8516-pwm diff --git a/Documentation/devicetree/bindings/pwm/pwm-amlogic.yaml b/Documentation/devicetree/bindings/pwm/pwm-amlogic.yaml index 527864a4d855..1d71d4f8f328 100644 --- a/Documentation/devicetree/bindings/pwm/pwm-amlogic.yaml +++ b/Documentation/devicetree/bindings/pwm/pwm-amlogic.yaml @@ -9,9 +9,6 @@ title: Amlogic PWM maintainers: - Heiner Kallweit -allOf: - - $ref: pwm.yaml# - properties: compatible: oneOf: @@ -24,31 +21,40 @@ properties: - amlogic,meson-g12a-ee-pwm - amlogic,meson-g12a-ao-pwm-ab - amlogic,meson-g12a-ao-pwm-cd - - amlogic,meson-s4-pwm + deprecated: true - items: - const: amlogic,meson-gx-pwm - const: amlogic,meson-gxbb-pwm + deprecated: true - items: - const: amlogic,meson-gx-ao-pwm - const: amlogic,meson-gxbb-ao-pwm + deprecated: true - items: - const: amlogic,meson8-pwm - const: amlogic,meson8b-pwm + deprecated: true + - enum: + - amlogic,meson8-pwm-v2 + - amlogic,meson-s4-pwm + - items: + - enum: + - amlogic,meson8b-pwm-v2 + - amlogic,meson-gxbb-pwm-v2 + - amlogic,meson-axg-pwm-v2 + - amlogic,meson-g12-pwm-v2 + - const: amlogic,meson8-pwm-v2 reg: maxItems: 1 clocks: minItems: 1 - maxItems: 2 + maxItems: 4 clock-names: - oneOf: - - items: - - enum: [clkin0, clkin1] - - items: - - const: clkin0 - - const: clkin1 + minItems: 1 + maxItems: 2 "#pwm-cells": const: 3 @@ -57,6 +63,79 @@ required: - compatible - reg +allOf: + - $ref: pwm.yaml# + + - if: + properties: + compatible: + contains: + enum: + - amlogic,meson8-pwm + - amlogic,meson8b-pwm + - amlogic,meson-gxbb-pwm + - amlogic,meson-gxbb-ao-pwm + - amlogic,meson-axg-ee-pwm + - amlogic,meson-axg-ao-pwm + - amlogic,meson-g12a-ee-pwm + - amlogic,meson-g12a-ao-pwm-ab + - amlogic,meson-g12a-ao-pwm-cd + then: + # Obsolete historic bindings tied to the driver implementation + # The clocks provided here are meant to be matched with the input + # known (hard-coded) in the driver and used to select pwm clock + # source. Currently, the linux driver ignores this. + # This is kept to maintain ABI backward compatibility. + properties: + clocks: + maxItems: 2 + clock-names: + oneOf: + - items: + - enum: [clkin0, clkin1] + - items: + - const: clkin0 + - const: clkin1 + + # Newer binding where clock describe the actual clock inputs of the pwm + # block. These are necessary but some inputs may be grounded. + - if: + properties: + compatible: + contains: + enum: + - amlogic,meson8-pwm-v2 + then: + properties: + clocks: + minItems: 1 + items: + - description: input clock 0 of the pwm block + - description: input clock 1 of the pwm block + - description: input clock 2 of the pwm block + - description: input clock 3 of the pwm block + clock-names: false + required: + - clocks + + # Newer IP block take a single input per channel, instead of 4 inputs + # for both channels + - if: + properties: + compatible: + contains: + enum: + - amlogic,meson-s4-pwm + then: + properties: + clocks: + items: + - description: input clock of PWM channel A + - description: input clock of PWM channel B + clock-names: false + required: + - clocks + additionalProperties: false examples: @@ -68,3 +147,17 @@ examples: clock-names = "clkin0", "clkin1"; #pwm-cells = <3>; }; + - | + pwm@2000 { + compatible = "amlogic,meson8-pwm-v2"; + reg = <0x1000 0x10>; + clocks = <&xtal>, <0>, <&fdiv4>, <&fdiv5>; + #pwm-cells = <3>; + }; + - | + pwm@1000 { + compatible = "amlogic,meson-s4-pwm"; + reg = <0x1000 0x10>; + clocks = <&pwm_src_a>, <&pwm_src_b>; + #pwm-cells = <3>; + }; diff --git a/Documentation/devicetree/bindings/pwm/pxa-pwm.txt b/Documentation/devicetree/bindings/pwm/pxa-pwm.txt deleted file mode 100644 index 5ae9f1e3c338..000000000000 --- a/Documentation/devicetree/bindings/pwm/pxa-pwm.txt +++ /dev/null @@ -1,30 +0,0 @@ -Marvell PWM controller - -Required properties: -- compatible: should be one or more of: - - "marvell,pxa250-pwm" - - "marvell,pxa270-pwm" - - "marvell,pxa168-pwm" - - "marvell,pxa910-pwm" -- reg: Physical base address and length of the registers used by the PWM channel - Note that one device instance must be created for each PWM that is used, so the - length covers only the register window for one PWM output, not that of the - entire PWM controller. Currently length is 0x10 for all supported devices. -- #pwm-cells: Should be 1. This cell is used to specify the period in - nanoseconds. - -Example PWM device node: - -pwm0: pwm@40b00000 { - compatible = "marvell,pxa250-pwm"; - reg = <0x40b00000 0x10>; - #pwm-cells = <1>; -}; - -Example PWM client node: - -backlight { - compatible = "pwm-backlight"; - pwms = <&pwm0 5000000>; - ... -} diff --git a/Documentation/devicetree/bindings/regulator/gpio-regulator.yaml b/Documentation/devicetree/bindings/regulator/gpio-regulator.yaml index f4c1f36e52e9..a34e85754658 100644 --- a/Documentation/devicetree/bindings/regulator/gpio-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/gpio-regulator.yaml @@ -47,6 +47,7 @@ properties: 1: HIGH Default is LOW if nothing else is specified. $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 maxItems: 8 items: enum: [0, 1] @@ -57,7 +58,8 @@ properties: regulator and matching GPIO configurations to achieve them. If there are no states in the "states" array, use a fixed regulator instead. $ref: /schemas/types.yaml#/definitions/uint32-matrix - maxItems: 8 + minItems: 2 + maxItems: 256 items: items: - description: Voltage in microvolts diff --git a/Documentation/devicetree/bindings/regulator/infineon,ir38060.yaml b/Documentation/devicetree/bindings/regulator/infineon,ir38060.yaml new file mode 100644 index 000000000000..e6ffbc2a2298 --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/infineon,ir38060.yaml @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/regulator/infineon,ir38060.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Infineon Buck Regulators with PMBUS interfaces + +maintainers: + - Not Me. + +allOf: + - $ref: regulator.yaml# + +properties: + compatible: + enum: + - infineon,ir38060 + - infineon,ir38064 + - infineon,ir38164 + - infineon,ir38263 + + reg: + maxItems: 1 + +required: + - compatible + - reg + +unevaluatedProperties: false + +examples: + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + + regulator@34 { + compatible = "infineon,ir38060"; + reg = <0x34>; + + regulator-min-microvolt = <437500>; + regulator-max-microvolt = <1387500>; + }; + }; diff --git a/Documentation/devicetree/bindings/regulator/mcp16502-regulator.txt b/Documentation/devicetree/bindings/regulator/mcp16502-regulator.txt deleted file mode 100644 index 451cc4e86b01..000000000000 --- a/Documentation/devicetree/bindings/regulator/mcp16502-regulator.txt +++ /dev/null @@ -1,144 +0,0 @@ -MCP16502 PMIC - -Required properties: -- compatible: "microchip,mcp16502" -- reg: I2C slave address -- lpm-gpios: GPIO for LPM pin. Note that this GPIO *must* remain high during - suspend-to-ram, keeping the PMIC into HIBERNATE mode; this - property is optional; -- regulators: A node that houses a sub-node for each regulator within - the device. Each sub-node is identified using the node's - name. The content of each sub-node is defined by the - standard binding for regulators; see regulator.txt. - -Regulators of MCP16502 PMIC: -1) VDD_IO - Buck (1.2 - 3.7 V) -2) VDD_DDR - Buck (0.6 - 1.85 V) -3) VDD_CORE - Buck (0.6 - 1.85 V) -4) VDD_OTHER - BUCK (0.6 - 1.85 V) -5) LDO1 - LDO (1.2 - 3.7 V) -6) LDO2 - LDO (1.2 - 3.7 V) - -Regulator modes: -2 - FPWM: higher precision, higher consumption -4 - AutoPFM: lower precision, lower consumption - -Each regulator is defined using the standard binding for regulators. - -Example: - -mcp16502@5b { - compatible = "microchip,mcp16502"; - reg = <0x5b>; - status = "okay"; - lpm-gpios = <&pioBU 7 GPIO_ACTIVE_HIGH>; - - regulators { - VDD_IO { - regulator-name = "VDD_IO"; - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <3700000>; - regulator-initial-mode = <2>; - regulator-allowed-modes = <2>, <4>; - regulator-always-on; - - regulator-state-standby { - regulator-on-in-suspend; - regulator-mode = <4>; - }; - - regulator-state-mem { - regulator-off-in-suspend; - regulator-mode = <4>; - }; - }; - - VDD_DDR { - regulator-name = "VDD_DDR"; - regulator-min-microvolt = <600000>; - regulator-max-microvolt = <1850000>; - regulator-initial-mode = <2>; - regulator-allowed-modes = <2>, <4>; - regulator-always-on; - - regulator-state-standby { - regulator-on-in-suspend; - regulator-mode = <4>; - }; - - regulator-state-mem { - regulator-on-in-suspend; - regulator-mode = <4>; - }; - }; - - VDD_CORE { - regulator-name = "VDD_CORE"; - regulator-min-microvolt = <600000>; - regulator-max-microvolt = <1850000>; - regulator-initial-mode = <2>; - regulator-allowed-modes = <2>, <4>; - regulator-always-on; - - regulator-state-standby { - regulator-on-in-suspend; - regulator-mode = <4>; - }; - - regulator-state-mem { - regulator-off-in-suspend; - regulator-mode = <4>; - }; - }; - - VDD_OTHER { - regulator-name = "VDD_OTHER"; - regulator-min-microvolt = <600000>; - regulator-max-microvolt = <1850000>; - regulator-initial-mode = <2>; - regulator-allowed-modes = <2>, <4>; - regulator-always-on; - - regulator-state-standby { - regulator-on-in-suspend; - regulator-mode = <4>; - }; - - regulator-state-mem { - regulator-off-in-suspend; - regulator-mode = <4>; - }; - }; - - LDO1 { - regulator-name = "LDO1"; - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <3700000>; - regulator-always-on; - - regulator-state-standby { - regulator-on-in-suspend; - }; - - regulator-state-mem { - regulator-off-in-suspend; - }; - }; - - LDO2 { - regulator-name = "LDO2"; - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <3700000>; - regulator-always-on; - - regulator-state-standby { - regulator-on-in-suspend; - }; - - regulator-state-mem { - regulator-off-in-suspend; - }; - }; - - }; -}; diff --git a/Documentation/devicetree/bindings/regulator/microchip,mcp16502.yaml b/Documentation/devicetree/bindings/regulator/microchip,mcp16502.yaml new file mode 100644 index 000000000000..1aca3646789e --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/microchip,mcp16502.yaml @@ -0,0 +1,180 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/regulator/microchip,mcp16502.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MCP16502 - High-Performance PMIC + +maintainers: + - Andrei Simion + +description: + The MCP16502 is an optimally integrated PMIC compatible + with Microchip's eMPUs(Embedded Microprocessor Units), + requiring Dynamic Voltage Scaling (DVS) with the use + of High-Performance mode (HPM). + +properties: + compatible: + const: microchip,mcp16502 + + lpm-gpios: + maxItems: 1 + description: GPIO for LPM pin. + Note that this GPIO must remain high during + suspend-to-ram, keeping the PMIC into HIBERNATE mode. + + reg: + maxItems: 1 + + regulators: + type: object + additionalProperties: false + description: List of regulators and its properties. + + patternProperties: + "^(VDD_(IO|CORE|DDR|OTHER)|LDO[1-2])$": + type: object + $ref: regulator.yaml# + unevaluatedProperties: false + + properties: + regulator-initial-mode: + enum: [2, 4] + default: 2 + description: Initial operating mode + + regulator-allowed-modes: + items: + enum: [2, 4] + description: Supported modes + 2 - FPWM higher precision, higher consumption + 4 - AutoPFM lower precision, lower consumption + +required: + - compatible + - reg + - regulators + +additionalProperties: false + +examples: + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + + pmic@5b { + compatible = "microchip,mcp16502"; + reg = <0x5b>; + + regulators { + VDD_IO { + regulator-name = "VDD_IO"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <2>; + regulator-allowed-modes = <2>, <4>; + regulator-always-on; + + regulator-state-standby { + regulator-on-in-suspend; + regulator-mode = <4>; + }; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-mode = <4>; + }; + }; + + VDD_DDR { + regulator-name = "VDD_DDR"; + regulator-min-microvolt = <1350000>; + regulator-max-microvolt = <1350000>; + regulator-initial-mode = <2>; + regulator-allowed-modes = <2>, <4>; + regulator-always-on; + + regulator-state-standby { + regulator-on-in-suspend; + regulator-mode = <4>; + }; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-mode = <4>; + }; + }; + + VDD_CORE { + regulator-name = "VDD_CORE"; + regulator-min-microvolt = <1150000>; + regulator-max-microvolt = <1150000>; + regulator-initial-mode = <2>; + regulator-allowed-modes = <2>, <4>; + regulator-always-on; + + regulator-state-standby { + regulator-on-in-suspend; + regulator-mode = <4>; + }; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-mode = <4>; + }; + }; + + VDD_OTHER { + regulator-name = "VDD_OTHER"; + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1250000>; + regulator-initial-mode = <2>; + regulator-allowed-modes = <2>, <4>; + regulator-always-on; + + regulator-state-standby { + regulator-on-in-suspend; + regulator-mode = <4>; + }; + + regulator-state-mem { + regulator-off-in-suspend; + regulator-mode = <4>; + }; + }; + + LDO1 { + regulator-name = "LDO1"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + + regulator-state-standby { + regulator-on-in-suspend; + }; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + LDO2 { + regulator-name = "LDO2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <3700000>; + regulator-always-on; + + regulator-state-standby { + regulator-on-in-suspend; + }; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml index 534f87e98716..8afb40c67af3 100644 --- a/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml @@ -19,8 +19,14 @@ allOf: properties: compatible: - enum: - - qcom,pm8150b-vbus-reg + oneOf: + - enum: + - qcom,pm8150b-vbus-reg + - items: + - enum: + - qcom,pm4125-vbus-reg + - qcom,pm6150-vbus-reg + - const: qcom,pm8150b-vbus-reg reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/regulator/ti,tps65132.yaml b/Documentation/devicetree/bindings/regulator/ti,tps65132.yaml new file mode 100644 index 000000000000..6a6d1a3d6fa7 --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/ti,tps65132.yaml @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/regulator/ti,tps65132.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: TI TPS65132 Dual Output Power Regulators + +maintainers: + - devicetree@vger.kernel.org + +description: | + The TPS65132 is designed to supply positive/negative driven applications. + + Datasheet is available at: + https://www.ti.com/lit/gpn/tps65132 + +properties: + compatible: + enum: + - ti,tps65132 + + reg: + maxItems: 1 + +patternProperties: + "^out[pn]$": + type: object + $ref: regulator.yaml# + unevaluatedProperties: false + description: + Properties for single regulator. + + properties: + enable-gpios: + maxItems: 1 + description: + GPIO specifier to enable the GPIO control (on/off) for regulator. + + active-discharge-gpios: + maxItems: 1 + description: + GPIO specifier to actively discharge the delay mechanism. + + ti,active-discharge-time-us: + description: Regulator active discharge time in microseconds. + + dependencies: + active-discharge-gpios: [ 'ti,active-discharge-time-us' ] + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + #include + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + regulator@3e { + compatible = "ti,tps65132"; + reg = <0x3e>; + + outp { + regulator-name = "outp"; + regulator-boot-on; + regulator-always-on; + enable-gpios = <&gpio 23 GPIO_ACTIVE_HIGH>; + }; + + outn { + regulator-name = "outn"; + regulator-boot-on; + regulator-always-on; + regulator-active-discharge = <0>; + enable-gpios = <&gpio 40 GPIO_ACTIVE_HIGH>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/regulator/tps65132-regulator.txt b/Documentation/devicetree/bindings/regulator/tps65132-regulator.txt deleted file mode 100644 index 3a3505520c69..000000000000 --- a/Documentation/devicetree/bindings/regulator/tps65132-regulator.txt +++ /dev/null @@ -1,46 +0,0 @@ -TPS65132 regulators - -Required properties: -- compatible: "ti,tps65132" -- reg: I2C slave address - -Optional Subnode: -Device supports two regulators OUTP and OUTN. A sub node within the - device node describe the properties of these regulators. The sub-node - names must be as follows: - -For regulator outp, the sub node name should be "outp". - -For regulator outn, the sub node name should be "outn". - --enable-gpios:(active high, output) Regulators are controlled by the input pins. - If it is connected to GPIO through host system then provide the - gpio number as per gpio.txt. --active-discharge-gpios: (active high, output) Some configurations use delay mechanisms - on the enable pin, to keep the regulator enabled for some time after - the enable signal goes low. This GPIO is used to actively discharge - the delay mechanism. Requires specification of ti,active-discharge-time-us --ti,active-discharge-time-us: how long the active discharge gpio should be - asserted for during active discharge, in microseconds. - -Each regulator is defined using the standard binding for regulators. - -Example: - - tps65132@3e { - compatible = "ti,tps65132"; - reg = <0x3e>; - - outp { - regulator-name = "outp"; - regulator-boot-on; - regulator-always-on; - enable-gpios = <&gpio 23 0>; - }; - - outn { - regulator-name = "outn"; - regulator-boot-on; - regulator-always-on; - regulator-active-discharge = <0>; - enable-gpios = <&gpio 40 0>; - }; - }; diff --git a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hdmi-blk-ctrl.yaml b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hdmi-blk-ctrl.yaml index 1be4ce2a45e8..bd1cdaa4f54b 100644 --- a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hdmi-blk-ctrl.yaml +++ b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hdmi-blk-ctrl.yaml @@ -27,8 +27,8 @@ properties: const: 1 power-domains: - minItems: 8 - maxItems: 8 + minItems: 10 + maxItems: 10 power-domain-names: items: @@ -40,10 +40,12 @@ properties: - const: trng - const: hdmi-tx - const: hdmi-tx-phy + - const: hdcp + - const: hrv clocks: - minItems: 4 - maxItems: 4 + minItems: 5 + maxItems: 5 clock-names: items: @@ -51,6 +53,7 @@ properties: - const: axi - const: ref_266m - const: ref_24m + - const: fdcc interconnects: maxItems: 3 @@ -82,12 +85,15 @@ examples: clocks = <&clk IMX8MP_CLK_HDMI_APB>, <&clk IMX8MP_CLK_HDMI_ROOT>, <&clk IMX8MP_CLK_HDMI_REF_266M>, - <&clk IMX8MP_CLK_HDMI_24M>; - clock-names = "apb", "axi", "ref_266m", "ref_24m"; + <&clk IMX8MP_CLK_HDMI_24M>, + <&clk IMX8MP_CLK_HDMI_FDCC_TST>; + clock-names = "apb", "axi", "ref_266m", "ref_24m", "fdcc"; power-domains = <&pgc_hdmimix>, <&pgc_hdmimix>, <&pgc_hdmimix>, <&pgc_hdmimix>, <&pgc_hdmimix>, <&pgc_hdmimix>, - <&pgc_hdmimix>, <&pgc_hdmi_phy>; + <&pgc_hdmimix>, <&pgc_hdmi_phy>, + <&pgc_hdmimix>, <&pgc_hdmimix>; power-domain-names = "bus", "irqsteer", "lcdif", "pai", "pvi", "trng", - "hdmi-tx", "hdmi-tx-phy"; + "hdmi-tx", "hdmi-tx-phy", + "hdcp", "hrv"; #power-domain-cells = <1>; }; diff --git a/Documentation/devicetree/bindings/spi/atmel,at91rm9200-spi.yaml b/Documentation/devicetree/bindings/spi/atmel,at91rm9200-spi.yaml index 58367587bfbc..32e7c14033c2 100644 --- a/Documentation/devicetree/bindings/spi/atmel,at91rm9200-spi.yaml +++ b/Documentation/devicetree/bindings/spi/atmel,at91rm9200-spi.yaml @@ -22,7 +22,6 @@ properties: - const: atmel,at91rm9200-spi - items: - const: microchip,sam9x7-spi - - const: microchip,sam9x60-spi - const: atmel,at91rm9200-spi reg: diff --git a/Documentation/devicetree/bindings/spi/samsung,spi.yaml b/Documentation/devicetree/bindings/spi/samsung,spi.yaml index 79da99ca0e53..f681372da81f 100644 --- a/Documentation/devicetree/bindings/spi/samsung,spi.yaml +++ b/Documentation/devicetree/bindings/spi/samsung,spi.yaml @@ -17,11 +17,13 @@ properties: compatible: oneOf: - enum: + - google,gs101-spi - samsung,s3c2443-spi # for S3C2443, S3C2416 and S3C2450 - samsung,s3c6410-spi - samsung,s5pv210-spi # for S5PV210 and S5PC110 - samsung,exynos4210-spi - samsung,exynos5433-spi + - samsung,exynos850-spi - samsung,exynosautov9-spi - tesla,fsd-spi - const: samsung,exynos7-spi @@ -74,8 +76,6 @@ required: - compatible - clocks - clock-names - - dmas - - dma-names - interrupts - reg diff --git a/Documentation/devicetree/bindings/spi/spi-controller.yaml b/Documentation/devicetree/bindings/spi/spi-controller.yaml index 524f6fe8c27b..093150c0cb87 100644 --- a/Documentation/devicetree/bindings/spi/spi-controller.yaml +++ b/Documentation/devicetree/bindings/spi/spi-controller.yaml @@ -69,6 +69,21 @@ properties: Should be generally avoided and be replaced by spi-cs-high + ACTIVE_HIGH. + fifo-depth: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Size of the RX and TX data FIFOs in bytes. + + rx-fifo-depth: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Size of the RX data FIFO in bytes. + + tx-fifo-depth: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Size of the TX data FIFO in bytes. + num-cs: $ref: /schemas/types.yaml#/definitions/uint32 description: @@ -116,6 +131,10 @@ patternProperties: - compatible - reg +dependencies: + rx-fifo-depth: [ tx-fifo-depth ] + tx-fifo-depth: [ rx-fifo-depth ] + allOf: - if: not: @@ -129,6 +148,14 @@ allOf: properties: "#address-cells": const: 0 + - not: + required: + - fifo-depth + - rx-fifo-depth + - not: + required: + - fifo-depth + - tx-fifo-depth additionalProperties: true diff --git a/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.yaml b/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.yaml index 727c5346b8ce..2ff174244795 100644 --- a/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.yaml +++ b/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.yaml @@ -22,6 +22,7 @@ properties: - enum: - fsl,imx8ulp-spi - fsl,imx93-spi + - fsl,imx95-spi - const: fsl,imx7ulp-spi reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/spi/spi-nxp-fspi.yaml b/Documentation/devicetree/bindings/spi/spi-nxp-fspi.yaml index 7fd591145480..4a5f41bde00f 100644 --- a/Documentation/devicetree/bindings/spi/spi-nxp-fspi.yaml +++ b/Documentation/devicetree/bindings/spi/spi-nxp-fspi.yaml @@ -15,12 +15,18 @@ allOf: properties: compatible: - enum: - - nxp,imx8dxl-fspi - - nxp,imx8mm-fspi - - nxp,imx8mp-fspi - - nxp,imx8qxp-fspi - - nxp,lx2160a-fspi + oneOf: + - enum: + - nxp,imx8dxl-fspi + - nxp,imx8mm-fspi + - nxp,imx8mp-fspi + - nxp,imx8qxp-fspi + - nxp,lx2160a-fspi + - items: + - enum: + - nxp,imx93-fspi + - nxp,imx95-fspi + - const: nxp,imx8mm-fspi reg: items: diff --git a/Documentation/devicetree/bindings/trivial-devices.yaml b/Documentation/devicetree/bindings/trivial-devices.yaml index 79dcd92c4a43..2210964faaf6 100644 --- a/Documentation/devicetree/bindings/trivial-devices.yaml +++ b/Documentation/devicetree/bindings/trivial-devices.yaml @@ -47,6 +47,8 @@ properties: - adi,lt7182s # AMS iAQ-Core VOC Sensor - ams,iaq-core + # Temperature monitoring of Astera Labs PT5161L PCIe retimer + - asteralabs,pt5161l # i2c serial eeprom (24cxx) - at,24c08 # ATSHA204 - i2c h/w symmetric crypto module @@ -129,6 +131,8 @@ properties: - mps,mp2975 # Monolithic Power Systems Inc. multi-phase hot-swap controller mp5990 - mps,mp5990 + # Monolithic Power Systems Inc. synchronous step-down converter mpq8785 + - mps,mpq8785 # Honeywell Humidicon HIH-6130 humidity/temperature sensor - honeywell,hi6130 # IBM Common Form Factor Power Supply Versions (all versions) @@ -139,14 +143,6 @@ properties: - ibm,cffps2 # Infineon IR36021 digital POL buck controller - infineon,ir36021 - # Infineon IR38060 Voltage Regulator - - infineon,ir38060 - # Infineon IR38064 Voltage Regulator - - infineon,ir38064 - # Infineon IR38164 Voltage Regulator - - infineon,ir38164 - # Infineon IR38263 Voltage Regulator - - infineon,ir38263 # Infineon IRPS5401 Voltage Regulator (PMIC) - infineon,irps5401 # Infineon TLV493D-A1B6 I2C 3D Magnetic Sensor diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index d371eb572164..6af0014835dd 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -109,6 +109,8 @@ patternProperties: description: Amlogic, Inc. "^ampere,.*": description: Ampere Computing LLC + "^amphenol,.*": + description: Amphenol Advanced Sensors "^ampire,.*": description: Ampire Co., Ltd. "^ams,.*": @@ -161,6 +163,8 @@ patternProperties: description: ASPEED Technology Inc. "^asrock,.*": description: ASRock Inc. + "^asteralabs,.*": + description: Astera Labs, Inc. "^asus,.*": description: AsusTek Computer Inc. "^atheros,.*": diff --git a/Documentation/driver-api/driver-model/devres.rst b/Documentation/driver-api/driver-model/devres.rst index c5f99d834ec5..7be8b8dd5f00 100644 --- a/Documentation/driver-api/driver-model/devres.rst +++ b/Documentation/driver-api/driver-model/devres.rst @@ -420,6 +420,7 @@ POWER devm_reboot_mode_unregister() PWM + devm_pwmchip_alloc() devm_pwmchip_add() devm_pwm_get() devm_fwnode_pwm_get() @@ -462,7 +463,7 @@ SLAVE DMA ENGINE SPI devm_spi_alloc_master() devm_spi_alloc_slave() - devm_spi_register_master() + devm_spi_register_controller() WATCHDOG devm_watchdog_register_device() diff --git a/Documentation/driver-api/gpio/consumer.rst b/Documentation/driver-api/gpio/consumer.rst index 3e588b9d678c..ab56ab0dd7a6 100644 --- a/Documentation/driver-api/gpio/consumer.rst +++ b/Documentation/driver-api/gpio/consumer.rst @@ -222,9 +222,9 @@ Use the following calls to access GPIOs from an atomic context:: int gpiod_get_value(const struct gpio_desc *desc); void gpiod_set_value(struct gpio_desc *desc, int value); -The values are boolean, zero for low, nonzero for high. When reading the value -of an output pin, the value returned should be what's seen on the pin. That -won't always match the specified output value, because of issues including +The values are boolean, zero for inactive, nonzero for active. When reading the +value of an output pin, the value returned should be what's seen on the pin. +That won't always match the specified output value, because of issues including open-drain signaling and output latencies. The get/set calls do not return errors because "invalid GPIO" should have been @@ -277,11 +277,11 @@ switch their output to a high impedance value. The consumer should not need to care. (For details read about open drain in driver.rst.) With this, all the gpiod_set_(array)_value_xxx() functions interpret the -parameter "value" as "asserted" ("1") or "de-asserted" ("0"). The physical line +parameter "value" as "active" ("1") or "inactive" ("0"). The physical line level will be driven accordingly. As an example, if the active low property for a dedicated GPIO is set, and the -gpiod_set_(array)_value_xxx() passes "asserted" ("1"), the physical line level +gpiod_set_(array)_value_xxx() passes "active" ("1"), the physical line level will be driven low. To summarize:: diff --git a/Documentation/driver-api/pwm.rst b/Documentation/driver-api/pwm.rst index 3c28ccc4b611..b41b1c56477f 100644 --- a/Documentation/driver-api/pwm.rst +++ b/Documentation/driver-api/pwm.rst @@ -143,11 +143,12 @@ to implement the pwm_*() functions itself. This means that it's impossible to have multiple PWM drivers in the system. For this reason it's mandatory for new drivers to use the generic PWM framework. -A new PWM controller/chip can be added using pwmchip_add() and removed -again with pwmchip_remove(). pwmchip_add() takes a filled in struct -pwm_chip as argument which provides a description of the PWM chip, the -number of PWM devices provided by the chip and the chip-specific -implementation of the supported PWM operations to the framework. +A new PWM controller/chip can be allocated using pwmchip_alloc(), then +registered using pwmchip_add() and removed again with pwmchip_remove(). To undo +pwmchip_alloc() use pwmchip_put(). pwmchip_add() takes a filled in struct +pwm_chip as argument which provides a description of the PWM chip, the number +of PWM devices provided by the chip and the chip-specific implementation of the +supported PWM operations to the framework. When implementing polarity support in a PWM driver, make sure to respect the signal conventions in the PWM framework. By definition, normal polarity diff --git a/Documentation/firmware-guide/acpi/index.rst b/Documentation/firmware-guide/acpi/index.rst index b6a42f4ffe03..b246902f523f 100644 --- a/Documentation/firmware-guide/acpi/index.rst +++ b/Documentation/firmware-guide/acpi/index.rst @@ -14,7 +14,6 @@ ACPI Support dsd/phy enumeration osi - method-customizing method-tracing DSD-properties-rules debug diff --git a/Documentation/firmware-guide/acpi/method-customizing.rst b/Documentation/firmware-guide/acpi/method-customizing.rst deleted file mode 100644 index de3ebcaed4cf..000000000000 --- a/Documentation/firmware-guide/acpi/method-customizing.rst +++ /dev/null @@ -1,89 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -======================================= -Linux ACPI Custom Control Method How To -======================================= - -:Author: Zhang Rui - - -Linux supports customizing ACPI control methods at runtime. - -Users can use this to: - -1. override an existing method which may not work correctly, - or just for debugging purposes. -2. insert a completely new method in order to create a missing - method such as _OFF, _ON, _STA, _INI, etc. - -For these cases, it is far simpler to dynamically install a single -control method rather than override the entire DSDT, because kernel -rebuild/reboot is not needed and test result can be got in minutes. - -.. note:: - - - Only ACPI METHOD can be overridden, any other object types like - "Device", "OperationRegion", are not recognized. Methods - declared inside scope operators are also not supported. - - - The same ACPI control method can be overridden for many times, - and it's always the latest one that used by Linux/kernel. - - - To get the ACPI debug object output (Store (AAAA, Debug)), - please run:: - - echo 1 > /sys/module/acpi/parameters/aml_debug_output - - -1. override an existing method -============================== -a) get the ACPI table via ACPI sysfs I/F. e.g. to get the DSDT, - just run "cat /sys/firmware/acpi/tables/DSDT > /tmp/dsdt.dat" -b) disassemble the table by running "iasl -d dsdt.dat". -c) rewrite the ASL code of the method and save it in a new file, -d) package the new file (psr.asl) to an ACPI table format. - Here is an example of a customized \_SB._AC._PSR method:: - - DefinitionBlock ("", "SSDT", 1, "", "", 0x20080715) - { - Method (\_SB_.AC._PSR, 0, NotSerialized) - { - Store ("In AC _PSR", Debug) - Return (ACON) - } - } - - Note that the full pathname of the method in ACPI namespace - should be used. -e) assemble the file to generate the AML code of the method. - e.g. "iasl -vw 6084 psr.asl" (psr.aml is generated as a result) - If parameter "-vw 6084" is not supported by your iASL compiler, - please try a newer version. -f) mount debugfs by "mount -t debugfs none /sys/kernel/debug" -g) override the old method via the debugfs by running - "cat /tmp/psr.aml > /sys/kernel/debug/acpi/custom_method" - -2. insert a new method -====================== -This is easier than overriding an existing method. -We just need to create the ASL code of the method we want to -insert and then follow the step c) ~ g) in section 1. - -3. undo your changes -==================== -The "undo" operation is not supported for a new inserted method -right now, i.e. we can not remove a method currently. -For an overridden method, in order to undo your changes, please -save a copy of the method original ASL code in step c) section 1, -and redo step c) ~ g) to override the method with the original one. - - -.. note:: We can use a kernel with multiple custom ACPI method running, - But each individual write to debugfs can implement a SINGLE - method override. i.e. if we want to insert/override multiple - ACPI methods, we need to redo step c) ~ g) for multiple times. - -.. note:: Be aware that root can mis-use this driver to modify arbitrary - memory and gain additional rights, if root's privileges got - restricted (for example if root is not allowed to load additional - modules after boot). diff --git a/Documentation/hwmon/aspeed-g6-pwm-tach.rst b/Documentation/hwmon/aspeed-g6-pwm-tach.rst new file mode 100644 index 000000000000..17398fe397fe --- /dev/null +++ b/Documentation/hwmon/aspeed-g6-pwm-tach.rst @@ -0,0 +1,26 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +Kernel driver aspeed-g6-pwm-tach +================================= + +Supported chips: + ASPEED AST2600 + +Authors: + + +Description: +------------ +This driver implements support for ASPEED AST2600 Fan Tacho controller. +The controller supports up to 16 tachometer inputs. + +The driver provides the following sensor accesses in sysfs: + +=============== ======= ====================================================== +fanX_input ro provide current fan rotation value in RPM as reported + by the fan to the device. +fanX_div rw Fan divisor: Supported value are power of 4 (1, 4, 16 + 64, ... 4194304) + The larger divisor, the less rpm accuracy and the less + affected by fan signal glitch. +=============== ======= ====================================================== diff --git a/Documentation/hwmon/asus_rog_ryujin.rst b/Documentation/hwmon/asus_rog_ryujin.rst new file mode 100644 index 000000000000..9f77da070022 --- /dev/null +++ b/Documentation/hwmon/asus_rog_ryujin.rst @@ -0,0 +1,47 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +Kernel driver asus_rog_ryujin +============================= + +Supported devices: + +* ASUS ROG RYUJIN II 360 + +Author: Aleksa Savic + +Description +----------- + +This driver enables hardware monitoring support for the listed ASUS ROG RYUJIN +all-in-one CPU liquid coolers. Available sensors are pump, internal and external +(controller) fan speed in RPM, their duties in PWM, as well as coolant temperature. + +Attaching external fans to the controller is optional and allows them to be +controlled from the device. If not connected, the fan-related sensors will +report zeroes. The controller is a separate hardware unit that comes bundled +with the AIO and connects to it to allow fan control. + +The addressable LCD screen is not supported in this driver and should +be controlled through userspace tools. + +Usage notes +----------- + +As these are USB HIDs, the driver can be loaded automatically by the kernel and +supports hot swapping. + +Sysfs entries +------------- + +=========== ============================================= +fan1_input Pump speed (in rpm) +fan2_input Internal fan speed (in rpm) +fan3_input External (controller) fan 1 speed (in rpm) +fan4_input External (controller) fan 2 speed (in rpm) +fan5_input External (controller) fan 3 speed (in rpm) +fan6_input External (controller) fan 4 speed (in rpm) +temp1_input Coolant temperature (in millidegrees Celsius) +pwm1 Pump duty +pwm2 Internal fan duty +pwm3 External (controller) fan duty +=========== ============================================= diff --git a/Documentation/hwmon/chipcap2.rst b/Documentation/hwmon/chipcap2.rst new file mode 100644 index 000000000000..dc165becc64c --- /dev/null +++ b/Documentation/hwmon/chipcap2.rst @@ -0,0 +1,73 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +Kernel driver ChipCap2 +====================== + +Supported chips: + + * Amphenol CC2D23, CC2D23S, CC2D25, CC2D25S, CC2D33, CC2D33S, CC2D35, CC2D35S + + Prefix: 'chipcap2' + + Addresses scanned: - + + Datasheet: https://www.amphenol-sensors.com/en/telaire/humidity/527-humidity-sensors/3095-chipcap-2 + +Author: + + - Javier Carrasco + +Description +----------- + +This driver implements support for the Amphenol ChipCap 2, a humidity and +temperature chip family. Temperature is measured in milli degrees celsius, +relative humidity is expressed as a per cent mille. The measurement ranges +are the following: + + - Relative humidity: 0 to 100000 pcm (14-bit resolution) + - Temperature: -40000 to +125000 m°C (14-bit resolution) + +The device communicates with the I2C protocol and uses the I2C address 0x28 +by default. + +Depending on the hardware configuration, up to two humidity alarms to control +minimum and maximum values are provided. Their thresholds and hystersis can be +configured via sysfs. + +Thresholds and hysteris must be provided as a per cent mille. These values +might be truncated to match the 14-bit device resolution (6.1 pcm/LSB) + +Known Issues +------------ + +The driver does not support I2C address and command window length modification. + +sysfs-Interface +--------------- + +The following list includes the sysfs attributes that the driver always provides, +their permissions and a short description: + +=============================== ======= ======================================== +Name Perm Description +=============================== ======= ======================================== +temp1_input: RO temperature input +humidity1_input: RO humidity input +=============================== ======= ======================================== + +The following list includes the sysfs attributes that the driver may provide +depending on the hardware configuration: + +=============================== ======= ======================================== +Name Perm Description +=============================== ======= ======================================== +humidity1_min: RW humidity low limit. Measurements under + this limit trigger a humidity low alarm +humidity1_max: RW humidity high limit. Measurements above + this limit trigger a humidity high alarm +humidity1_min_hyst: RW humidity low hystersis +humidity1_max_hyst: RW humidity high hystersis +humidity1_min_alarm: RO humidity low alarm indicator +humidity1_max_alarm: RO humidity high alarm indicator +=============================== ======= ======================================== diff --git a/Documentation/hwmon/emc2305.rst b/Documentation/hwmon/emc2305.rst index 2403dbaf2728..d0bfffe46358 100644 --- a/Documentation/hwmon/emc2305.rst +++ b/Documentation/hwmon/emc2305.rst @@ -6,7 +6,6 @@ Kernel driver emc2305 Supported chips: Microchip EMC2305, EMC2303, EMC2302, EMC2301 - Addresses scanned: I2C 0x27, 0x2c, 0x2d, 0x2e, 0x2f, 0x4c, 0x4d Prefixes: 'emc2305' Datasheet: Publicly available at the Microchip website : diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst index c7ed1f73ac06..1ca7a4fe1f8f 100644 --- a/Documentation/hwmon/index.rst +++ b/Documentation/hwmon/index.rst @@ -44,13 +44,16 @@ Hardware Monitoring Kernel Drivers aquacomputer_d5next asb100 asc7621 + aspeed-g6-pwm-tach aspeed-pwm-tacho asus_ec_sensors + asus_rog_ryujin asus_wmi_sensors bcm54140 bel-pfe bpa-rs600 bt1-pvt + chipcap2 coretemp corsair-cpro corsair-psu @@ -129,6 +132,7 @@ Hardware Monitoring Kernel Drivers ltc4245 ltc4260 ltc4261 + ltc4282 ltc4286 max127 max15301 @@ -163,6 +167,7 @@ Hardware Monitoring Kernel Drivers mp2975 mp5023 mp5990 + mpq8785 nct6683 nct6775 nct7802 @@ -171,6 +176,7 @@ Hardware Monitoring Kernel Drivers nsa320 ntc_thermistor nzxt-kraken2 + nzxt-kraken3 nzxt-smart2 occ oxp-sensors @@ -185,6 +191,7 @@ Hardware Monitoring Kernel Drivers pmbus powerz powr1220 + pt5161l pxe1610 pwm-fan q54sj108a2 @@ -208,6 +215,7 @@ Hardware Monitoring Kernel Drivers smsc47m1 sparx5-temp stpddc60 + surface_fan sy7636a-hwmon tc654 tc74 diff --git a/Documentation/hwmon/ltc4282.rst b/Documentation/hwmon/ltc4282.rst new file mode 100644 index 000000000000..a87ec3564998 --- /dev/null +++ b/Documentation/hwmon/ltc4282.rst @@ -0,0 +1,133 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Kernel drivers ltc4282 +========================================== + +Supported chips: + + * Analog Devices LTC4282 + + Prefix: 'ltc4282' + + Addresses scanned: - I2C 0x40 - 0x5A (7-bit) + Addresses scanned: - I2C 0x80 - 0xB4 with a step of 2 (8-bit) + + Datasheet: + + https://www.analog.com/media/en/technical-documentation/data-sheets/ltc4282.pdf + +Author: Nuno Sá + +Description +___________ + +The LTC4282 hot swap controller allows a board to be safely inserted and removed +from a live backplane. Using one or more external N-channel pass transistors, +board supply voltage and inrush current are ramped up at an adjustable rate. An +I2C interface and onboard ADC allows for monitoring of board current, voltage, +power, energy and fault status. The device features analog foldback current +limiting and supply monitoring for applications from 2.9V to 33V. Dual 12V gate +drive allows high power applications to either share safe operating area across +parallel MOSFETs or support a 2-stage start-up that first charges the load +capacitance followed by enabling a low on-resistance path to the load. The +LTC4282 is well suited to high power applications because the precise monitoring +capability and accurate current limiting reduce the extremes in which both loads +and power supplies must safely operate. Non-volatile configuration allows for +flexibility in the autonomous generation of alerts and response to faults. + +Sysfs entries +_____________ + +The following attributes are supported. Limits are read-write and all the other +attributes are read-only. Note that in0 and in1 are mutually exclusive. Enabling +one disables the other and disabling one enables the other. + +======================= ========================================== +in0_input Output voltage (mV). +in0_min Undervoltage threshold +in0_max Overvoltage threshold +in0_lowest Lowest measured voltage +in0_highest Highest measured voltage +in0_reset_history Write 1 to reset in0 history. + Also clears fet bad and short fault logs. +in0_min_alarm Undervoltage alarm +in0_max_alarm Overvoltage alarm +in0_enable Enable/Disable VSOURCE monitoring +in0_fault Failure in the MOSFETs. Either bad or shorted FET. +in0_label Channel label (VSOURCE) + +in1_input Input voltage (mV). +in1_min Undervoltage threshold +in1_max Overvoltage threshold +in1_lowest Lowest measured voltage +in1_highest Highest measured voltage +in1_reset_history Write 1 to reset in1 history. + Also clears over/undervoltage fault logs. +in1_min_alarm Undervoltage alarm +in1_max_alarm Overvoltage alarm +in1_lcrit_alarm Critical Undervoltage alarm +in1_crit_alarm Critical Overvoltage alarm +in1_enable Enable/Disable VDD monitoring +in1_label Channel label (VDD) + +in2_input GPIO voltage (mV) +in2_min Undervoltage threshold +in2_max Overvoltage threshold +in2_lowest Lowest measured voltage +in2_highest Highest measured voltage +in2_reset_history Write 1 to reset in2 history +in2_min_alarm Undervoltage alarm +in2_max_alarm Overvoltage alarm +in2_label Channel label (VGPIO) + +curr1_input Sense current (mA) +curr1_min Undercurrent threshold +curr1_max Overcurrent threshold +curr1_lowest Lowest measured current +curr1_highest Highest measured current +curr1_reset_history Write 1 to reset curr1 history. + Also clears overcurrent fault logs. +curr1_min_alarm Undercurrent alarm +curr1_max_alarm Overcurrent alarm +curr1_crit_alarm Critical Overcurrent alarm +curr1_label Channel label (ISENSE) + +power1_input Power (in uW) +power1_min Low power threshold +power1_max High power threshold +power1_input_lowest Historical minimum power use +power1_input_highest Historical maximum power use +power1_reset_history Write 1 to reset power1 history. + Also clears power bad fault logs. +power1_min_alarm Low power alarm +power1_max_alarm High power alarm +power1_label Channel label (Power) + +energy1_input Measured energy over time (in microJoule) +energy1_enable Enable/Disable Energy accumulation +======================= ========================================== + +DebugFs entries +_______________ + +The chip also has a fault log register where failures can be logged. Hence, +as these are logging events, we give access to them in debugfs. Note that +even if some failure is detected in these logs, it does necessarily mean +that the failure is still present. As mentioned in the proper Sysfs entries, +these logs can be cleared by writing in the proper reset_history attribute. + +.. warning:: The debugfs interface is subject to change without notice + and is only available when the kernel is compiled with + ``CONFIG_DEBUG_FS`` defined. + +``/sys/kernel/debug/ltc4282-hwmon[X]/`` +contains the following attributes: + +======================= ========================================== +power1_bad_fault_log Set to 1 by a power1 bad fault occurring. +in0_fet_short_fault_log Set to 1 when the ADC detects a FET-short fault. +in0_fet_bad_fault_log Set to 1 when a FET-BAD fault occurs. +in1_crit_fault_log Set to 1 by a VDD overvoltage fault occurring. +in1_lcrit_fault_log Set to 1 by a VDD undervoltage fault occurring. +curr1_crit_fault_log Set to 1 by an overcurrent fault occurring. +======================= ========================================== diff --git a/Documentation/hwmon/max6620.rst b/Documentation/hwmon/max6620.rst index 84c1c44d3de4..d70173bf0242 100644 --- a/Documentation/hwmon/max6620.rst +++ b/Documentation/hwmon/max6620.rst @@ -11,7 +11,7 @@ Supported chips: Addresses scanned: none - Datasheet: http://pdfserv.maxim-ic.com/en/ds/MAX6620.pdf + Datasheet: https://www.analog.com/media/en/technical-documentation/data-sheets/max6620.pdf Authors: - L\. Grunenberg diff --git a/Documentation/hwmon/mpq8785.rst b/Documentation/hwmon/mpq8785.rst new file mode 100644 index 000000000000..bf8176b87086 --- /dev/null +++ b/Documentation/hwmon/mpq8785.rst @@ -0,0 +1,94 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Kernel driver mpq8785 +======================= + +Supported chips: + + * MPS MPQ8785 + + Prefix: 'mpq8785' + +Author: Charles Hsu + +Description +----------- + +The MPQ8785 is a fully integrated, PMBus-compatible, high-frequency, synchronous +buck converter. The MPQ8785 offers a very compact solution that achieves up to +40A output current per phase, with excellent load and line regulation over a +wide input supply range. The MPQ8785 operates at high efficiency over a wide +output current load range. + +The PMBus interface provides converter configurations and key parameters +monitoring. + +The MPQ8785 adopts MPS's proprietary multi-phase digital constant-on-time (MCOT) +control, which provides fast transient response and eases loop stabilization. +The MCOT scheme also allows multiple MPQ8785 devices to be connected in parallel +with excellent current sharing and phase interleaving for high-current +applications. + +Fully integrated protection features include over-current protection (OCP), +over-voltage protection (OVP), under-voltage protection (UVP), and +over-temperature protection (OTP). + +The MPQ8785 requires a minimal number of readily available, standard external +components, and is available in a TLGA (5mmx6mm) package. + +Device compliant with: + +- PMBus rev 1.3 interface. + +The driver exports the following attributes via the 'sysfs' files +for input voltage: + +**in1_input** + +**in1_label** + +**in1_max** + +**in1_max_alarm** + +**in1_min** + +**in1_min_alarm** + +**in1_crit** + +**in1_crit_alarm** + +The driver provides the following attributes for output voltage: + +**in2_input** + +**in2_label** + +**in2_alarm** + +The driver provides the following attributes for output current: + +**curr1_input** + +**curr1_label** + +**curr1_max** + +**curr1_max_alarm** + +**curr1_crit** + +**curr1_crit_alarm** + +The driver provides the following attributes for temperature: + +**temp1_input** + +**temp1_max** + +**temp1_max_alarm** + +**temp1_crit** + +**temp1_crit_alarm** diff --git a/Documentation/hwmon/nct6683.rst b/Documentation/hwmon/nct6683.rst index 3e7f6ee779c2..2a7a78eb1b46 100644 --- a/Documentation/hwmon/nct6683.rst +++ b/Documentation/hwmon/nct6683.rst @@ -64,4 +64,5 @@ Intel DB85FL NCT6683D EC firmware version 1.0 build 04/03/13 ASRock X570 NCT6683D EC firmware version 1.0 build 06/28/19 ASRock X670E NCT6686D EC firmware version 1.0 build 05/19/22 MSI B550 NCT6687D EC firmware version 1.0 build 05/07/20 +MSI X670-P NCT6687D EC firmware version 0.0 build 09/27/22 =============== =============================================== diff --git a/Documentation/hwmon/nzxt-kraken3.rst b/Documentation/hwmon/nzxt-kraken3.rst new file mode 100644 index 000000000000..90fd9dec15ff --- /dev/null +++ b/Documentation/hwmon/nzxt-kraken3.rst @@ -0,0 +1,74 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +Kernel driver nzxt-kraken3 +========================== + +Supported devices: + +* NZXT Kraken X53 +* NZXT Kraken X63 +* NZXT Kraken X73 +* NZXT Kraken Z53 +* NZXT Kraken Z63 +* NZXT Kraken Z73 + +Author: Jonas Malaco, Aleksa Savic + +Description +----------- + +This driver enables hardware monitoring support for NZXT Kraken X53/X63/X73 and +Z53/Z63/Z73 all-in-one CPU liquid coolers. All models expose liquid temperature +and pump speed (in RPM), as well as PWM control (either as a fixed value +or through a temp-PWM curve). The Z-series models additionally expose the speed +and duty of an optionally connected fan, with the same PWM control capabilities. + +Pump and fan duty control mode can be set through pwm[1-2]_enable, where 1 is +for the manual control mode and 2 is for the liquid temp to PWM curve mode. +Writing a 0 disables control of the channel through the driver after setting its +duty to 100%. + +The temperature of the curves relates to the fixed [20-59] range, correlating to +the detected liquid temperature. Only PWM values (ranging from 0-255) can be set. +If in curve mode, setting point values should be done in moderation - the devices +require complete curves to be sent for each change; they can lock up or discard +the changes if they are too numerous at once. Suggestion is to set them while +in an another mode, and then apply them by switching to curve. + +The devices can report if they are faulty. The driver supports that situation +and will issue a warning. This can also happen when the USB cable is connected, +but SATA power is not. + +The addressable RGB LEDs and LCD screen (only on Z-series models) are not +supported in this driver, but can be controlled through existing userspace tools, +such as `liquidctl`_. + +.. _liquidctl: https://github.com/liquidctl/liquidctl + +Usage Notes +----------- + +As these are USB HIDs, the driver can be loaded automatically by the kernel and +supports hot swapping. + +Possible pwm_enable values are: + +====== ========================================================================== +0 Set fan to 100% +1 Direct PWM mode (applies value in corresponding PWM entry) +2 Curve control mode (applies the temp-PWM duty curve based on coolant temp) +====== ========================================================================== + +Sysfs entries +------------- + +============================== ================================================================ +fan1_input Pump speed (in rpm) +fan2_input Fan speed (in rpm) +temp1_input Coolant temperature (in millidegrees Celsius) +pwm1 Pump duty (value between 0-255) +pwm1_enable Pump duty control mode (0: disabled, 1: manual, 2: curve) +pwm2 Fan duty (value between 0-255) +pwm2_enable Fan duty control mode (0: disabled, 1: manual, 2: curve) +temp[1-2]_auto_point[1-40]_pwm Temp-PWM duty curves (for pump and fan), related to coolant temp +============================== ================================================================ diff --git a/Documentation/hwmon/oxp-sensors.rst b/Documentation/hwmon/oxp-sensors.rst index 3adeb7406243..55b1ef61625e 100644 --- a/Documentation/hwmon/oxp-sensors.rst +++ b/Documentation/hwmon/oxp-sensors.rst @@ -33,6 +33,7 @@ Currently the driver supports the following handhelds: - AOK ZOE A1 PRO - Aya Neo 2 - Aya Neo AIR + - Aya Neo AIR Plus (Mendocino) - Aya Neo AIR Pro - Aya Neo Geek - OneXPlayer AMD diff --git a/Documentation/hwmon/pt5161l.rst b/Documentation/hwmon/pt5161l.rst new file mode 100644 index 000000000000..1b97336991ea --- /dev/null +++ b/Documentation/hwmon/pt5161l.rst @@ -0,0 +1,42 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +Kernel driver pt5161l +===================== + +Supported chips: + + * Astera Labs PT5161L + + Prefix: 'pt5161l' + + Addresses scanned: I2C 0x20 - 0x27 + + Datasheet: Not publicly available. + +Authors: Cosmo Chou + +Description +----------- + +This driver implements support for temperature monitoring of Astera Labs +PT5161L series PCIe retimer chips. + +This driver implementation originates from the CSDK available at +https://github.com/facebook/openbmc/tree/helium/common/recipes-lib/retimer-v2.14 +The communication protocol utilized is based on the I2C/SMBus standard. + +Sysfs entries +---------------- + +================ ============================================== +temp1_input Measured temperature (in millidegrees Celsius) +================ ============================================== + +Debugfs entries +---------------- + +================ =============================== +fw_load_status Firmware load status +fw_ver Firmware version of the retimer +heartbeat_status Heartbeat status +================ =============================== diff --git a/Documentation/hwmon/sht3x.rst b/Documentation/hwmon/sht3x.rst index 957c854f5d08..9585fa7c5a5d 100644 --- a/Documentation/hwmon/sht3x.rst +++ b/Documentation/hwmon/sht3x.rst @@ -65,6 +65,10 @@ When the temperature and humidity readings move back between the hysteresis values, the alert bit is set to 0 and the alert pin on the sensor is set to low. +The serial number exposed to debugfs allows for unique identification of the +sensors. For sts32, sts33 and sht33, the manufacturer provides calibration +certificates through an API. + sysfs-Interface --------------- @@ -99,3 +103,10 @@ repeatability: write or read repeatability, higher repeatability means - 1: medium repeatability - 2: high repeatability =================== ============================================================ + +debugfs-Interface +----------------- + +=================== ============================================================ +serial_number: unique serial number of the sensor in decimal +=================== ============================================================ diff --git a/Documentation/hwmon/surface_fan.rst b/Documentation/hwmon/surface_fan.rst new file mode 100644 index 000000000000..07942574c4f0 --- /dev/null +++ b/Documentation/hwmon/surface_fan.rst @@ -0,0 +1,25 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +Kernel driver surface_fan +========================= + +Supported Devices: + + * Microsoft Surface Pro 9 + +Author: Ivor Wanders + +Description +----------- + +This provides monitoring of the fan found in some Microsoft Surface Pro devices, +like the Surface Pro 9. The fan is always controlled by the onboard controller. + +Sysfs interface +--------------- + +======================= ======= ========================================= +Name Perm Description +======================= ======= ========================================= +``fan1_input`` RO Current fan speed in RPM. +======================= ======= ========================================= diff --git a/Documentation/mm/slub.rst b/Documentation/mm/slub.rst index be75971532f5..b517ee28a955 100644 --- a/Documentation/mm/slub.rst +++ b/Documentation/mm/slub.rst @@ -9,7 +9,7 @@ SLUB can enable debugging only for selected slabs in order to avoid an impact on overall system performance which may make a bug more difficult to find. -In order to switch debugging on one can add an option ``slub_debug`` +In order to switch debugging on one can add an option ``slab_debug`` to the kernel command line. That will enable full debugging for all slabs. @@ -26,16 +26,16 @@ be enabled on the command line. F.e. no tracking information will be available without debugging on and validation can only partially be performed if debugging was not switched on. -Some more sophisticated uses of slub_debug: +Some more sophisticated uses of slab_debug: ------------------------------------------- -Parameters may be given to ``slub_debug``. If none is specified then full +Parameters may be given to ``slab_debug``. If none is specified then full debugging is enabled. Format: -slub_debug= +slab_debug= Enable options for all slabs -slub_debug=,,,... +slab_debug=,,,... Enable options only for select slabs (no spaces after a comma) @@ -60,23 +60,23 @@ Possible debug options are:: F.e. in order to boot just with sanity checks and red zoning one would specify:: - slub_debug=FZ + slab_debug=FZ Trying to find an issue in the dentry cache? Try:: - slub_debug=,dentry + slab_debug=,dentry to only enable debugging on the dentry cache. You may use an asterisk at the end of the slab name, in order to cover all slabs with the same prefix. For example, here's how you can poison the dentry cache as well as all kmalloc slabs:: - slub_debug=P,kmalloc-*,dentry + slab_debug=P,kmalloc-*,dentry Red zoning and tracking may realign the slab. We can just apply sanity checks to the dentry cache with:: - slub_debug=F,dentry + slab_debug=F,dentry Debugging options may require the minimum possible slab order to increase as a result of storing the metadata (for example, caches with PAGE_SIZE object @@ -84,20 +84,20 @@ sizes). This has a higher liklihood of resulting in slab allocation errors in low memory situations or if there's high fragmentation of memory. To switch off debugging for such caches by default, use:: - slub_debug=O + slab_debug=O You can apply different options to different list of slab names, using blocks of options. This will enable red zoning for dentry and user tracking for kmalloc. All other slabs will not get any debugging enabled:: - slub_debug=Z,dentry;U,kmalloc-* + slab_debug=Z,dentry;U,kmalloc-* You can also enable options (e.g. sanity checks and poisoning) for all caches except some that are deemed too performance critical and don't need to be debugged by specifying global debug options followed by a list of slab names with "-" as options:: - slub_debug=FZ;-,zs_handle,zspage + slab_debug=FZ;-,zs_handle,zspage The state of each debug option for a slab can be found in the respective files under:: @@ -105,7 +105,7 @@ under:: /sys/kernel/slab// If the file contains 1, the option is enabled, 0 means disabled. The debug -options from the ``slub_debug`` parameter translate to the following files:: +options from the ``slab_debug`` parameter translate to the following files:: F sanity_checks Z red_zone @@ -129,7 +129,7 @@ in order to reduce overhead and increase cache hotness of objects. Slab validation =============== -SLUB can validate all object if the kernel was booted with slub_debug. In +SLUB can validate all object if the kernel was booted with slab_debug. In order to do so you must have the ``slabinfo`` tool. Then you can do :: @@ -150,29 +150,29 @@ list_lock once in a while to deal with partial slabs. That overhead is governed by the order of the allocation for each slab. The allocations can be influenced by kernel parameters: -.. slub_min_objects=x (default 4) -.. slub_min_order=x (default 0) -.. slub_max_order=x (default 3 (PAGE_ALLOC_COSTLY_ORDER)) +.. slab_min_objects=x (default: automatically scaled by number of cpus) +.. slab_min_order=x (default 0) +.. slab_max_order=x (default 3 (PAGE_ALLOC_COSTLY_ORDER)) -``slub_min_objects`` +``slab_min_objects`` allows to specify how many objects must at least fit into one slab in order for the allocation order to be acceptable. In general slub will be able to perform this number of allocations on a slab without consulting centralized resources (list_lock) where contention may occur. -``slub_min_order`` +``slab_min_order`` specifies a minimum order of slabs. A similar effect like - ``slub_min_objects``. + ``slab_min_objects``. -``slub_max_order`` - specified the order at which ``slub_min_objects`` should no +``slab_max_order`` + specified the order at which ``slab_min_objects`` should no longer be checked. This is useful to avoid SLUB trying to - generate super large order pages to fit ``slub_min_objects`` + generate super large order pages to fit ``slab_min_objects`` of a slab cache with large object sizes into one high order page. Setting command line parameter ``debug_guardpage_minorder=N`` (N > 0), forces setting - ``slub_max_order`` to 0, what cause minimum possible order of + ``slab_max_order`` to 0, what cause minimum possible order of slabs allocation. SLUB Debug output @@ -219,7 +219,7 @@ Here is a sample of slub debug output:: FIX kmalloc-8: Restoring Redzone 0xc90f6d28-0xc90f6d2b=0xcc If SLUB encounters a corrupted object (full detection requires the kernel -to be booted with slub_debug) then the following output will be dumped +to be booted with slab_debug) then the following output will be dumped into the syslog: 1. Description of the problem encountered @@ -239,7 +239,7 @@ into the syslog: pid= (Object allocation / free information is only available if SLAB_STORE_USER is - set for the slab. slub_debug sets that option) + set for the slab. slab_debug sets that option) 2. The object contents if an object was involved. @@ -262,7 +262,7 @@ into the syslog: the object boundary. (Redzone information is only available if SLAB_RED_ZONE is set. - slub_debug sets that option) + slab_debug sets that option) Padding
: Unused data to fill up the space in order to get the next object @@ -296,7 +296,7 @@ Emergency operations Minimal debugging (sanity checks alone) can be enabled by booting with:: - slub_debug=F + slab_debug=F This will be generally be enough to enable the resiliency features of slub which will keep the system running even if a bad kernel component will @@ -311,13 +311,13 @@ and enabling debugging only for that cache I.e.:: - slub_debug=F,dentry + slab_debug=F,dentry If the corruption occurs by writing after the end of the object then it may be advisable to enable a Redzone to avoid corrupting the beginning of other objects:: - slub_debug=FZ,dentry + slab_debug=FZ,dentry Extended slabinfo mode and plotting =================================== diff --git a/Documentation/power/energy-model.rst b/Documentation/power/energy-model.rst index 13225965c9a4..ada4938c37e5 100644 --- a/Documentation/power/energy-model.rst +++ b/Documentation/power/energy-model.rst @@ -71,6 +71,31 @@ whose performance is scaled together. Performance domains generally have a required to have the same micro-architecture. CPUs in different performance domains can have different micro-architectures. +To better reflect power variation due to static power (leakage) the EM +supports runtime modifications of the power values. The mechanism relies on +RCU to free the modifiable EM perf_state table memory. Its user, the task +scheduler, also uses RCU to access this memory. The EM framework provides +API for allocating/freeing the new memory for the modifiable EM table. +The old memory is freed automatically using RCU callback mechanism when there +are no owners anymore for the given EM runtime table instance. This is tracked +using kref mechanism. The device driver which provided the new EM at runtime, +should call EM API to free it safely when it's no longer needed. The EM +framework will handle the clean-up when it's possible. + +The kernel code which want to modify the EM values is protected from concurrent +access using a mutex. Therefore, the device driver code must run in sleeping +context when it tries to modify the EM. + +With the runtime modifiable EM we switch from a 'single and during the entire +runtime static EM' (system property) design to a 'single EM which can be +changed during runtime according e.g. to the workload' (system and workload +property) design. + +It is possible also to modify the CPU performance values for each EM's +performance state. Thus, the full power and performance profile (which +is an exponential curve) can be changed according e.g. to the workload +or system property. + 2. Core APIs ------------ @@ -175,10 +200,82 @@ CPUfreq governor is in use in case of CPU device. Currently this calculation is not provided for other type of devices. More details about the above APIs can be found in ```` -or in Section 2.4 +or in Section 2.5 -2.4 Description details of this API +2.4 Runtime modifications +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Drivers willing to update the EM at runtime should use the following dedicated +function to allocate a new instance of the modified EM. The API is listed +below:: + + struct em_perf_table __rcu *em_table_alloc(struct em_perf_domain *pd); + +This allows to allocate a structure which contains the new EM table with +also RCU and kref needed by the EM framework. The 'struct em_perf_table' +contains array 'struct em_perf_state state[]' which is a list of performance +states in ascending order. That list must be populated by the device driver +which wants to update the EM. The list of frequencies can be taken from +existing EM (created during boot). The content in the 'struct em_perf_state' +must be populated by the driver as well. + +This is the API which does the EM update, using RCU pointers swap:: + + int em_dev_update_perf_domain(struct device *dev, + struct em_perf_table __rcu *new_table); + +Drivers must provide a pointer to the allocated and initialized new EM +'struct em_perf_table'. That new EM will be safely used inside the EM framework +and will be visible to other sub-systems in the kernel (thermal, powercap). +The main design goal for this API is to be fast and avoid extra calculations +or memory allocations at runtime. When pre-computed EMs are available in the +device driver, than it should be possible to simply re-use them with low +performance overhead. + +In order to free the EM, provided earlier by the driver (e.g. when the module +is unloaded), there is a need to call the API:: + + void em_table_free(struct em_perf_table __rcu *table); + +It will allow the EM framework to safely remove the memory, when there is +no other sub-system using it, e.g. EAS. + +To use the power values in other sub-systems (like thermal, powercap) there is +a need to call API which protects the reader and provide consistency of the EM +table data:: + + struct em_perf_state *em_perf_state_from_pd(struct em_perf_domain *pd); + +It returns the 'struct em_perf_state' pointer which is an array of performance +states in ascending order. +This function must be called in the RCU read lock section (after the +rcu_read_lock()). When the EM table is not needed anymore there is a need to +call rcu_real_unlock(). In this way the EM safely uses the RCU read section +and protects the users. It also allows the EM framework to manage the memory +and free it. More details how to use it can be found in Section 3.2 in the +example driver. + +There is dedicated API for device drivers to calculate em_perf_state::cost +values:: + + int em_dev_compute_costs(struct device *dev, struct em_perf_state *table, + int nr_states); + +These 'cost' values from EM are used in EAS. The new EM table should be passed +together with the number of entries and device pointer. When the computation +of the cost values is done properly the return value from the function is 0. +The function takes care for right setting of inefficiency for each performance +state as well. It updates em_perf_state::flags accordingly. +Then such prepared new EM can be passed to the em_dev_update_perf_domain() +function, which will allow to use it. + +More details about the above APIs can be found in ```` +or in Section 3.2 with an example code showing simple implementation of the +updating mechanism in a device driver. + + +2.5 Description details of this API ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. kernel-doc:: include/linux/energy_model.h :internal: @@ -187,8 +284,11 @@ or in Section 2.4 :export: -3. Example driver ------------------ +3. Examples +----------- + +3.1 Example driver with EM registration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The CPUFreq framework supports dedicated callback for registering the EM for a given CPU(s) 'policy' object: cpufreq_driver::register_em(). @@ -242,3 +342,78 @@ EM framework:: 39 static struct cpufreq_driver foo_cpufreq_driver = { 40 .register_em = foo_cpufreq_register_em, 41 }; + + +3.2 Example driver with EM modification +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This section provides a simple example of a thermal driver modifying the EM. +The driver implements a foo_thermal_em_update() function. The driver is woken +up periodically to check the temperature and modify the EM data:: + + -> drivers/soc/example/example_em_mod.c + + 01 static void foo_get_new_em(struct foo_context *ctx) + 02 { + 03 struct em_perf_table __rcu *em_table; + 04 struct em_perf_state *table, *new_table; + 05 struct device *dev = ctx->dev; + 06 struct em_perf_domain *pd; + 07 unsigned long freq; + 08 int i, ret; + 09 + 10 pd = em_pd_get(dev); + 11 if (!pd) + 12 return; + 13 + 14 em_table = em_table_alloc(pd); + 15 if (!em_table) + 16 return; + 17 + 18 new_table = em_table->state; + 19 + 20 rcu_read_lock(); + 21 table = em_perf_state_from_pd(pd); + 22 for (i = 0; i < pd->nr_perf_states; i++) { + 23 freq = table[i].frequency; + 24 foo_get_power_perf_values(dev, freq, &new_table[i]); + 25 } + 26 rcu_read_unlock(); + 27 + 28 /* Calculate 'cost' values for EAS */ + 29 ret = em_dev_compute_costs(dev, table, pd->nr_perf_states); + 30 if (ret) { + 31 dev_warn(dev, "EM: compute costs failed %d\n", ret); + 32 em_free_table(em_table); + 33 return; + 34 } + 35 + 36 ret = em_dev_update_perf_domain(dev, em_table); + 37 if (ret) { + 38 dev_warn(dev, "EM: update failed %d\n", ret); + 39 em_free_table(em_table); + 40 return; + 41 } + 42 + 43 /* + 44 * Since it's one-time-update drop the usage counter. + 45 * The EM framework will later free the table when needed. + 46 */ + 47 em_table_free(em_table); + 48 } + 49 + 50 /* + 51 * Function called periodically to check the temperature and + 52 * update the EM if needed + 53 */ + 54 static void foo_thermal_em_update(struct foo_context *ctx) + 55 { + 56 struct device *dev = ctx->dev; + 57 int cpu; + 58 + 59 ctx->temperature = foo_get_temp(dev, ctx); + 60 if (ctx->temperature < FOO_EM_UPDATE_TEMP_THRESHOLD) + 61 return; + 62 + 63 foo_get_new_em(ctx); + 64 } diff --git a/Documentation/power/opp.rst b/Documentation/power/opp.rst index a7c03c470980..1b7f1d854f14 100644 --- a/Documentation/power/opp.rst +++ b/Documentation/power/opp.rst @@ -305,7 +305,7 @@ dev_pm_opp_get_opp_count { /* Do things */ num_available = dev_pm_opp_get_opp_count(dev); - speeds = kzalloc(sizeof(u32) * num_available, GFP_KERNEL); + speeds = kcalloc(num_available, sizeof(u32), GFP_KERNEL); /* populate the table in increasing order */ freq = 0; while (!IS_ERR(opp = dev_pm_opp_find_freq_ceil(dev, &freq))) { diff --git a/Documentation/power/pci.rst b/Documentation/power/pci.rst index a125544b4cb6..12070320307e 100644 --- a/Documentation/power/pci.rst +++ b/Documentation/power/pci.rst @@ -625,7 +625,7 @@ The PCI subsystem-level callbacks they correspond to:: pci_pm_poweroff() pci_pm_poweroff_noirq() -work in analogy with pci_pm_suspend() and pci_pm_poweroff_noirq(), respectively, +work in analogy with pci_pm_suspend() and pci_pm_suspend_noirq(), respectively, although they don't attempt to save the device's standard configuration registers. diff --git a/Documentation/power/runtime_pm.rst b/Documentation/power/runtime_pm.rst index 65b86e487afe..5c4e730f38d0 100644 --- a/Documentation/power/runtime_pm.rst +++ b/Documentation/power/runtime_pm.rst @@ -154,7 +154,7 @@ suspending the device are satisfied) and to queue up a suspend request for the device in that case. If there is no idle callback, or if the callback returns 0, then the PM core will attempt to carry out a runtime suspend of the device, also respecting devices configured for autosuspend. In essence this means a -call to pm_runtime_autosuspend() (do note that drivers needs to update the +call to __pm_runtime_autosuspend() (do note that drivers needs to update the device last busy mark, pm_runtime_mark_last_busy(), to control the delay under this circumstance). To prevent this (for example, if the callback routine has started a delayed suspend), the routine must return a non-zero value. Negative @@ -396,10 +396,9 @@ drivers/base/power/runtime.c and include/linux/pm_runtime.h: nonzero, increment the counter and return 1; otherwise return 0 without changing the counter - `int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count);` + `int pm_runtime_get_if_active(struct device *dev);` - return -EINVAL if 'power.disable_depth' is nonzero; otherwise, if the - runtime PM status is RPM_ACTIVE, and either ign_usage_count is true - or the device's usage_count is non-zero, increment the counter and + runtime PM status is RPM_ACTIVE, increment the counter and return 1; otherwise return 0 without changing the counter `void pm_runtime_put_noidle(struct device *dev);` @@ -410,6 +409,10 @@ drivers/base/power/runtime.c and include/linux/pm_runtime.h: pm_request_idle(dev) and return its result `int pm_runtime_put_autosuspend(struct device *dev);` + - does the same as __pm_runtime_put_autosuspend() for now, but in the + future, will also call pm_runtime_mark_last_busy() as well, DO NOT USE! + + `int __pm_runtime_put_autosuspend(struct device *dev);` - decrement the device's usage counter; if the result is 0 then run pm_request_autosuspend(dev) and return its result @@ -540,6 +543,7 @@ It is safe to execute the following helper functions from interrupt context: - pm_runtime_put_noidle() - pm_runtime_put() - pm_runtime_put_autosuspend() +- __pm_runtime_put_autosuspend() - pm_runtime_enable() - pm_suspend_ignore_children() - pm_runtime_set_active() @@ -730,6 +734,7 @@ out the following operations: for it, respectively. 7. Generic subsystem callbacks +============================== Subsystems may wish to conserve code space by using the set of generic power management callbacks provided by the PM core, defined in @@ -865,9 +870,9 @@ automatically be delayed until the desired period of inactivity has elapsed. Inactivity is determined based on the power.last_busy field. Drivers should call pm_runtime_mark_last_busy() to update this field after carrying out I/O, -typically just before calling pm_runtime_put_autosuspend(). The desired length -of the inactivity period is a matter of policy. Subsystems can set this length -initially by calling pm_runtime_set_autosuspend_delay(), but after device +typically just before calling __pm_runtime_put_autosuspend(). The desired +length of the inactivity period is a matter of policy. Subsystems can set this +length initially by calling pm_runtime_set_autosuspend_delay(), but after device registration the length should be controlled by user space, using the /sys/devices/.../power/autosuspend_delay_ms attribute. @@ -878,7 +883,7 @@ instead of the non-autosuspend counterparts:: Instead of: pm_runtime_suspend use: pm_runtime_autosuspend; Instead of: pm_schedule_suspend use: pm_request_autosuspend; - Instead of: pm_runtime_put use: pm_runtime_put_autosuspend; + Instead of: pm_runtime_put use: __pm_runtime_put_autosuspend; Instead of: pm_runtime_put_sync use: pm_runtime_put_sync_autosuspend. Drivers may also continue to use the non-autosuspend helper functions; they @@ -917,7 +922,7 @@ Here is a schematic pseudo-code example:: lock(&foo->private_lock); if (--foo->num_pending_requests == 0) { pm_runtime_mark_last_busy(&foo->dev); - pm_runtime_put_autosuspend(&foo->dev); + __pm_runtime_put_autosuspend(&foo->dev); } else { foo_process_next_request(foo); } diff --git a/Documentation/spi/spi-summary.rst b/Documentation/spi/spi-summary.rst index 33f05901ccf3..546de37d6caf 100644 --- a/Documentation/spi/spi-summary.rst +++ b/Documentation/spi/spi-summary.rst @@ -9,7 +9,7 @@ What is SPI? The "Serial Peripheral Interface" (SPI) is a synchronous four wire serial link used to connect microcontrollers to sensors, memory, and peripherals. It's a simple "de facto" standard, not complicated enough to acquire a -standardization body. SPI uses a master/slave configuration. +standardization body. SPI uses a host/target configuration. The three signal wires hold a clock (SCK, often on the order of 10 MHz), and parallel data lines with "Master Out, Slave In" (MOSI) or "Master In, @@ -19,14 +19,14 @@ commonly used. Each clock cycle shifts data out and data in; the clock doesn't cycle except when there is a data bit to shift. Not all data bits are used though; not every protocol uses those full duplex capabilities. -SPI masters use a fourth "chip select" line to activate a given SPI slave +SPI hosts use a fourth "chip select" line to activate a given SPI target device, so those three signal wires may be connected to several chips -in parallel. All SPI slaves support chipselects; they are usually active -low signals, labeled nCSx for slave 'x' (e.g. nCS0). Some devices have -other signals, often including an interrupt to the master. +in parallel. All SPI targets support chipselects; they are usually active +low signals, labeled nCSx for target 'x' (e.g. nCS0). Some devices have +other signals, often including an interrupt to the host. Unlike serial busses like USB or SMBus, even low level protocols for -SPI slave functions are usually not interoperable between vendors +SPI target functions are usually not interoperable between vendors (except for commodities like SPI memory chips). - SPI may be used for request/response style device protocols, as with @@ -43,10 +43,10 @@ SPI slave functions are usually not interoperable between vendors - Sometimes SPI is used to daisy-chain devices, like shift registers. -In the same way, SPI slaves will only rarely support any kind of automatic -discovery/enumeration protocol. The tree of slave devices accessible from -a given SPI master will normally be set up manually, with configuration -tables. +In the same way, SPI targets will only rarely support any kind of automatic +discovery/enumeration protocol. The tree of target devices accessible from +a given SPI host controller will normally be set up manually, with +configuration tables. SPI is only one of the names used by such four-wire protocols, and most controllers have no problem handling "MicroWire" (think of it as @@ -62,8 +62,8 @@ course they won't handle full duplex transfers. You may find such chips described as using "three wire" signaling: SCK, data, nCSx. (That data line is sometimes called MOMI or SISO.) -Microcontrollers often support both master and slave sides of the SPI -protocol. This document (and Linux) supports both the master and slave +Microcontrollers often support both host and target sides of the SPI +protocol. This document (and Linux) supports both the host and target sides of SPI interactions. @@ -75,7 +75,7 @@ protocol supported by every MMC or SD memory card. (The older "DataFlash" cards, predating MMC cards but using the same connectors and card shape, support only SPI.) Some PC hardware uses SPI flash for BIOS code. -SPI slave chips range from digital/analog converters used for analog +SPI target chips range from digital/analog converters used for analog sensors and codecs, to memory, to peripherals like USB controllers or Ethernet adapters; and more. @@ -118,8 +118,8 @@ starting low (CPOL=0) and data stabilized for sampling during the trailing clock edge (CPHA=1), that's SPI mode 1. Note that the clock mode is relevant as soon as the chipselect goes -active. So the master must set the clock to inactive before selecting -a slave, and the slave can tell the chosen polarity by sampling the +active. So the host must set the clock to inactive before selecting +a target, and the target can tell the chosen polarity by sampling the clock level when its select line goes active. That's why many devices support for example both modes 0 and 3: they don't care about polarity, and always clock data in/out on rising clock edges. @@ -142,13 +142,13 @@ There are two types of SPI driver, here called: Controller drivers ... controllers may be built into System-On-Chip - processors, and often support both Master and Slave roles. + processors, and often support both Controller and target roles. These drivers touch hardware registers and may use DMA. Or they can be PIO bitbangers, needing just GPIO pins. Protocol drivers ... these pass messages through the controller - driver to communicate with a Slave or Master device on the + driver to communicate with a target or Controller device on the other side of an SPI link. So for example one protocol driver might talk to the MTD layer to export @@ -179,22 +179,22 @@ shows up in sysfs in several locations:: /sys/bus/spi/drivers/D ... driver for one or more spi*.* devices /sys/class/spi_master/spiB ... symlink to a logical node which could hold - class related state for the SPI master controller managing bus "B". + class related state for the SPI host controller managing bus "B". All spiB.* devices share one physical SPI bus segment, with SCLK, MOSI, and MISO. /sys/devices/.../CTLR/slave ... virtual file for (un)registering the - slave device for an SPI slave controller. - Writing the driver name of an SPI slave handler to this file - registers the slave device; writing "(null)" unregisters the slave + target device for an SPI target controller. + Writing the driver name of an SPI target handler to this file + registers the target device; writing "(null)" unregisters the target device. - Reading from this file shows the name of the slave device ("(null)" + Reading from this file shows the name of the target device ("(null)" if not registered). /sys/class/spi_slave/spiB ... symlink to a logical node which could hold - class related state for the SPI slave controller on bus "B". When + class related state for the SPI target controller on bus "B". When registered, a single spiB.* device is present here, possible sharing - the physical SPI bus segment with other SPI slave devices. + the physical SPI bus segment with other SPI target devices. At this time, the only class-specific state is the bus number ("B" in "spiB"), so those /sys/class entries are only useful to quickly identify busses. @@ -270,10 +270,10 @@ same SOC controller is used. For example, on one board SPI might use an external clock, where another derives the SPI clock from current settings of some master clock. -Declare Slave Devices -^^^^^^^^^^^^^^^^^^^^^ +Declare target Devices +^^^^^^^^^^^^^^^^^^^^^^ -The second kind of information is a list of what SPI slave devices exist +The second kind of information is a list of what SPI target devices exist on the target board, often with some board-specific data needed for the driver to work correctly. @@ -316,7 +316,7 @@ sharing a bus with a device that interprets chipselect "backwards" is not possible until the infrastructure knows how to deselect it. Then your board initialization code would register that table with the SPI -infrastructure, so that it's available later when the SPI master controller +infrastructure, so that it's available later when the SPI host controller driver is registered:: spi_register_board_info(spi_board_info, ARRAY_SIZE(spi_board_info)); @@ -469,39 +469,39 @@ routines are available to allocate and zero-initialize an spi_message with several transfers. -How do I write an "SPI Master Controller Driver"? +How do I write an "SPI Controller Driver"? ------------------------------------------------- An SPI controller will probably be registered on the platform_bus; write a driver to bind to the device, whichever bus is involved. -The main task of this type of driver is to provide an "spi_master". -Use spi_alloc_master() to allocate the master, and spi_master_get_devdata() -to get the driver-private data allocated for that device. +The main task of this type of driver is to provide an "spi_controller". +Use spi_alloc_host() to allocate the host controller, and +spi_controller_get_devdata() to get the driver-private data allocated for that +device. :: - struct spi_master *master; + struct spi_controller *ctlr; struct CONTROLLER *c; - master = spi_alloc_master(dev, sizeof *c); - if (!master) + ctlr = spi_alloc_host(dev, sizeof *c); + if (!ctlr) return -ENODEV; - c = spi_master_get_devdata(master); + c = spi_controller_get_devdata(ctlr); -The driver will initialize the fields of that spi_master, including the -bus number (maybe the same as the platform device ID) and three methods -used to interact with the SPI core and SPI protocol drivers. It will -also initialize its own internal state. (See below about bus numbering -and those methods.) +The driver will initialize the fields of that spi_controller, including the bus +number (maybe the same as the platform device ID) and three methods used to +interact with the SPI core and SPI protocol drivers. It will also initialize +its own internal state. (See below about bus numbering and those methods.) -After you initialize the spi_master, then use spi_register_master() to +After you initialize the spi_controller, then use spi_register_controller() to publish it to the rest of the system. At that time, device nodes for the controller and any predeclared spi devices will be made available, and the driver model core will take care of binding them to drivers. -If you need to remove your SPI controller driver, spi_unregister_master() -will reverse the effect of spi_register_master(). +If you need to remove your SPI controller driver, spi_unregister_controller() +will reverse the effect of spi_register_controller(). Bus Numbering @@ -519,49 +519,49 @@ then be replaced by a dynamically assigned number. You'd then need to treat this as a non-static configuration (see above). -SPI Master Methods -^^^^^^^^^^^^^^^^^^ +SPI Host Controller Methods +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -``master->setup(struct spi_device *spi)`` +``ctlr->setup(struct spi_device *spi)`` This sets up the device clock rate, SPI mode, and word sizes. Drivers may change the defaults provided by board_info, and then call spi_setup(spi) to invoke this routine. It may sleep. - Unless each SPI slave has its own configuration registers, don't + Unless each SPI target has its own configuration registers, don't change them right away ... otherwise drivers could corrupt I/O that's in progress for other SPI devices. .. note:: BUG ALERT: for some reason the first version of - many spi_master drivers seems to get this wrong. + many spi_controller drivers seems to get this wrong. When you code setup(), ASSUME that the controller is actively processing transfers for another device. -``master->cleanup(struct spi_device *spi)`` +``ctlr->cleanup(struct spi_device *spi)`` Your controller driver may use spi_device.controller_state to hold state it dynamically associates with that device. If you do that, be sure to provide the cleanup() method to free that state. -``master->prepare_transfer_hardware(struct spi_master *master)`` +``ctlr->prepare_transfer_hardware(struct spi_controller *ctlr)`` This will be called by the queue mechanism to signal to the driver that a message is coming in soon, so the subsystem requests the driver to prepare the transfer hardware by issuing this call. This may sleep. -``master->unprepare_transfer_hardware(struct spi_master *master)`` +``ctlr->unprepare_transfer_hardware(struct spi_controller *ctlr)`` This will be called by the queue mechanism to signal to the driver that there are no more messages pending in the queue and it may relax the hardware (e.g. by power management calls). This may sleep. -``master->transfer_one_message(struct spi_master *master, struct spi_message *mesg)`` +``ctlr->transfer_one_message(struct spi_controller *ctlr, struct spi_message *mesg)`` The subsystem calls the driver to transfer a single message while queuing transfers that arrive in the meantime. When the driver is finished with this message, it must call spi_finalize_current_message() so the subsystem can issue the next message. This may sleep. -``master->transfer_one(struct spi_master *master, struct spi_device *spi, struct spi_transfer *transfer)`` +``ctrl->transfer_one(struct spi_controller *ctlr, struct spi_device *spi, struct spi_transfer *transfer)`` The subsystem calls the driver to transfer a single transfer while queuing transfers that arrive in the meantime. When the driver is finished with this transfer, it must call @@ -576,15 +576,15 @@ SPI Master Methods * 0: transfer is finished * 1: transfer is still in progress -``master->set_cs_timing(struct spi_device *spi, u8 setup_clk_cycles, u8 hold_clk_cycles, u8 inactive_clk_cycles)`` - This method allows SPI client drivers to request SPI master controller +``ctrl->set_cs_timing(struct spi_device *spi, u8 setup_clk_cycles, u8 hold_clk_cycles, u8 inactive_clk_cycles)`` + This method allows SPI client drivers to request SPI host controller for configuring device specific CS setup, hold and inactive timing requirements. Deprecated Methods ^^^^^^^^^^^^^^^^^^ -``master->transfer(struct spi_device *spi, struct spi_message *message)`` +``ctrl->transfer(struct spi_device *spi, struct spi_message *message)`` This must not sleep. Its responsibility is to arrange that the transfer happens and its complete() callback is issued. The two will normally happen later, after other transfers complete, and diff --git a/Documentation/translations/zh_CN/power/opp.rst b/Documentation/translations/zh_CN/power/opp.rst index 8d6e3f6f6202..7470fa2d4c43 100644 --- a/Documentation/translations/zh_CN/power/opp.rst +++ b/Documentation/translations/zh_CN/power/opp.rst @@ -274,7 +274,7 @@ dev_pm_opp_get_opp_count { /* 做一些事情 */ num_available = dev_pm_opp_get_opp_count(dev); - speeds = kzalloc(sizeof(u32) * num_available, GFP_KERNEL); + speeds = kcalloc(num_available, sizeof(u32), GFP_KERNEL); /* 按升序填充表 */ freq = 0; while (!IS_ERR(opp = dev_pm_opp_find_freq_ceil(dev, &freq))) { diff --git a/Documentation/userspace-api/gpio/chardev.rst b/Documentation/userspace-api/gpio/chardev.rst new file mode 100644 index 000000000000..c58dd9771ac9 --- /dev/null +++ b/Documentation/userspace-api/gpio/chardev.rst @@ -0,0 +1,116 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=================================== +GPIO Character Device Userspace API +=================================== + +This is latest version (v2) of the character device API, as defined in +``include/uapi/linux/gpio.h.`` + +First added in 5.10. + +.. note:: + Do NOT abuse userspace APIs to control hardware that has proper kernel + drivers. There may already be a driver for your use case, and an existing + kernel driver is sure to provide a superior solution to bitbashing + from userspace. + + Read Documentation/driver-api/gpio/drivers-on-gpio.rst to avoid reinventing + kernel wheels in userspace. + + Similarly, for multi-function lines there may be other subsystems, such as + Documentation/spi/index.rst, Documentation/i2c/index.rst, + Documentation/driver-api/pwm.rst, Documentation/w1/index.rst etc, that + provide suitable drivers and APIs for your hardware. + +Basic examples using the character device API can be found in ``tools/gpio/*``. + +The API is based around two major objects, the :ref:`gpio-v2-chip` and the +:ref:`gpio-v2-line-request`. + +.. _gpio-v2-chip: + +Chip +==== + +The Chip represents a single GPIO chip and is exposed to userspace using device +files of the form ``/dev/gpiochipX``. + +Each chip supports a number of GPIO lines, +:c:type:`chip.lines`. Lines on the chip are identified by an +``offset`` in the range from 0 to ``chip.lines - 1``, i.e. `[0,chip.lines)`. + +Lines are requested from the chip using gpio-v2-get-line-ioctl.rst +and the resulting line request is used to access the GPIO chip's lines or +monitor the lines for edge events. + +Within this documentation, the file descriptor returned by calling `open()` +on the GPIO device file is referred to as ``chip_fd``. + +Operations +---------- + +The following operations may be performed on the chip: + +.. toctree:: + :titlesonly: + + Get Line + Get Chip Info + Get Line Info + Watch Line Info + Unwatch Line Info + Read Line Info Changed Events + +.. _gpio-v2-line-request: + +Line Request +============ + +Line requests are created by gpio-v2-get-line-ioctl.rst and provide +access to a set of requested lines. The line request is exposed to userspace +via the anonymous file descriptor returned in +:c:type:`request.fd` by gpio-v2-get-line-ioctl.rst. + +Within this documentation, the line request file descriptor is referred to +as ``req_fd``. + +Operations +---------- + +The following operations may be performed on the line request: + +.. toctree:: + :titlesonly: + + Get Line Values + Set Line Values + Read Line Edge Events + Reconfigure Lines + +Types +===== + +This section contains the structs and enums that are referenced by the API v2, +as defined in ``include/uapi/linux/gpio.h``. + +.. kernel-doc:: include/uapi/linux/gpio.h + :identifiers: + gpio_v2_line_attr_id + gpio_v2_line_attribute + gpio_v2_line_changed_type + gpio_v2_line_config + gpio_v2_line_config_attribute + gpio_v2_line_event + gpio_v2_line_event_id + gpio_v2_line_flag + gpio_v2_line_info + gpio_v2_line_info_changed + gpio_v2_line_request + gpio_v2_line_values + gpiochip_info + +.. toctree:: + :hidden: + + error-codes diff --git a/Documentation/userspace-api/gpio/chardev_v1.rst b/Documentation/userspace-api/gpio/chardev_v1.rst new file mode 100644 index 000000000000..67124b1d0487 --- /dev/null +++ b/Documentation/userspace-api/gpio/chardev_v1.rst @@ -0,0 +1,131 @@ +.. SPDX-License-Identifier: GPL-2.0 + +======================================== +GPIO Character Device Userspace API (v1) +======================================== + +.. warning:: + This API is obsoleted by chardev.rst (v2). + + New developments should use the v2 API, and existing developments are + encouraged to migrate as soon as possible, as this API will be removed + in the future. The v2 API is a functional superset of the v1 API so any + v1 call can be directly translated to a v2 equivalent. + + This interface will continue to be maintained for the migration period, + but new features will only be added to the new API. + +First added in 4.8. + +The API is based around three major objects, the :ref:`gpio-v1-chip`, the +:ref:`gpio-v1-line-handle`, and the :ref:`gpio-v1-line-event`. + +Where "line event" is used in this document it refers to the request that can +monitor a line for edge events, not the edge events themselves. + +.. _gpio-v1-chip: + +Chip +==== + +The Chip represents a single GPIO chip and is exposed to userspace using device +files of the form ``/dev/gpiochipX``. + +Each chip supports a number of GPIO lines, +:c:type:`chip.lines`. Lines on the chip are identified by an +``offset`` in the range from 0 to ``chip.lines - 1``, i.e. `[0,chip.lines)`. + +Lines are requested from the chip using either gpio-get-linehandle-ioctl.rst +and the resulting line handle is used to access the GPIO chip's lines, or +gpio-get-lineevent-ioctl.rst and the resulting line event is used to monitor +a GPIO line for edge events. + +Within this documentation, the file descriptor returned by calling `open()` +on the GPIO device file is referred to as ``chip_fd``. + +Operations +---------- + +The following operations may be performed on the chip: + +.. toctree:: + :titlesonly: + + Get Line Handle + Get Line Event + Get Chip Info + Get Line Info + Watch Line Info + Unwatch Line Info + Read Line Info Changed Events + +.. _gpio-v1-line-handle: + +Line Handle +=========== + +Line handles are created by gpio-get-linehandle-ioctl.rst and provide +access to a set of requested lines. The line handle is exposed to userspace +via the anonymous file descriptor returned in +:c:type:`request.fd` by gpio-get-linehandle-ioctl.rst. + +Within this documentation, the line handle file descriptor is referred to +as ``handle_fd``. + +Operations +---------- + +The following operations may be performed on the line handle: + +.. toctree:: + :titlesonly: + + Get Line Values + Set Line Values + Reconfigure Lines + +.. _gpio-v1-line-event: + +Line Event +========== + +Line events are created by gpio-get-lineevent-ioctl.rst and provide +access to a requested line. The line event is exposed to userspace +via the anonymous file descriptor returned in +:c:type:`request.fd` by gpio-get-lineevent-ioctl.rst. + +Within this documentation, the line event file descriptor is referred to +as ``event_fd``. + +Operations +---------- + +The following operations may be performed on the line event: + +.. toctree:: + :titlesonly: + + Get Line Value + Read Line Edge Events + +Types +===== + +This section contains the structs that are referenced by the ABI v1. + +The :c:type:`struct gpiochip_info` is common to ABI v1 and v2. + +.. kernel-doc:: include/uapi/linux/gpio.h + :identifiers: + gpioevent_data + gpioevent_request + gpiohandle_config + gpiohandle_data + gpiohandle_request + gpioline_info + gpioline_info_changed + +.. toctree:: + :hidden: + + error-codes diff --git a/Documentation/userspace-api/gpio/error-codes.rst b/Documentation/userspace-api/gpio/error-codes.rst new file mode 100644 index 000000000000..6bf2948990cd --- /dev/null +++ b/Documentation/userspace-api/gpio/error-codes.rst @@ -0,0 +1,79 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _gpio_errors: + +******************* +GPIO Error Codes +******************* + +.. _gpio-errors: + +.. tabularcolumns:: |p{2.5cm}|p{15.0cm}| + +.. flat-table:: Common GPIO error codes + :header-rows: 0 + :stub-columns: 0 + :widths: 1 16 + + - - ``EAGAIN`` (aka ``EWOULDBLOCK``) + + - The device was opened in non-blocking mode and a read can't + be performed as there is no data available. + + - - ``EBADF`` + + - The file descriptor is not valid. + + - - ``EBUSY`` + + - The ioctl can't be handled because the device is busy. Typically + returned when an ioctl attempts something that would require the + usage of a resource that was already allocated. The ioctl must not + be retried without performing another action to fix the problem + first. + + - - ``EFAULT`` + + - There was a failure while copying data from/to userspace, probably + caused by an invalid pointer reference. + + - - ``EINVAL`` + + - One or more of the ioctl parameters are invalid or out of the + allowed range. This is a widely used error code. + + - - ``ENODEV`` + + - Device not found or was removed. + + - - ``ENOMEM`` + + - There's not enough memory to handle the desired operation. + + - - ``EPERM`` + + - Permission denied. Typically returned in response to an attempt + to perform an action incompatible with the current line + configuration. + + - - ``EIO`` + + - I/O error. Typically returned when there are problems communicating + with a hardware device or requesting features that hardware does not + support. This could indicate broken or flaky hardware. + It's a 'Something is wrong, I give up!' type of error. + + - - ``ENXIO`` + + - Typically returned when a feature requiring interrupt support was + requested, but the line does not support interrupts. + +.. note:: + + #. This list is not exhaustive; ioctls may return other error codes. + Since errors may have side effects such as a driver reset, + applications should abort on unexpected errors, or otherwise + assume that the device is in a bad state. + + #. Request-specific error codes are listed in the individual + requests descriptions. diff --git a/Documentation/userspace-api/gpio/gpio-get-chipinfo-ioctl.rst b/Documentation/userspace-api/gpio/gpio-get-chipinfo-ioctl.rst new file mode 100644 index 000000000000..05f07fdefe2f --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-get-chipinfo-ioctl.rst @@ -0,0 +1,41 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_GET_CHIPINFO_IOCTL: + +*********************** +GPIO_GET_CHIPINFO_IOCTL +*********************** + +Name +==== + +GPIO_GET_CHIPINFO_IOCTL - Get the publicly available information for a chip. + +Synopsis +======== + +.. c:macro:: GPIO_GET_CHIPINFO_IOCTL + +``int ioctl(int chip_fd, GPIO_GET_CHIPINFO_IOCTL, struct gpiochip_info *info)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``info`` + The :c:type:`chip_info` to be populated. + +Description +=========== + +Gets the publicly available information for a particular GPIO chip. + +Return Value +============ + +On success 0 and ``info`` is populated with the chip info. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-get-lineevent-ioctl.rst b/Documentation/userspace-api/gpio/gpio-get-lineevent-ioctl.rst new file mode 100644 index 000000000000..09a9254f38cf --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-get-lineevent-ioctl.rst @@ -0,0 +1,84 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_GET_LINEEVENT_IOCTL: + +************************ +GPIO_GET_LINEEVENT_IOCTL +************************ + +.. warning:: + This ioctl is part of chardev_v1.rst and is obsoleted by + gpio-v2-get-line-ioctl.rst. + +Name +==== + +GPIO_GET_LINEEVENT_IOCTL - Request a line with edge detection from the kernel. + +Synopsis +======== + +.. c:macro:: GPIO_GET_LINEEVENT_IOCTL + +``int ioctl(int chip_fd, GPIO_GET_LINEEVENT_IOCTL, struct gpioevent_request *request)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``request`` + The :c:type:`event_request` specifying the line + to request and its configuration. + +Description +=========== + +Request a line with edge detection from the kernel. + +On success, the requesting process is granted exclusive access to the line +value and may receive events when edges are detected on the line, as +described in gpio-lineevent-data-read.rst. + +The state of a line is guaranteed to remain as requested until the returned +file descriptor is closed. Once the file descriptor is closed, the state of +the line becomes uncontrolled from the userspace perspective, and may revert +to its default state. + +Requesting a line already in use is an error (**EBUSY**). + +Requesting edge detection on a line that does not support interrupts is an +error (**ENXIO**). + +As with the :ref:`line handle`, the +bias configuration is best effort. + +Closing the ``chip_fd`` has no effect on existing line events. + +Configuration Rules +------------------- + +The following configuration rules apply: + +The line event is requested as an input, so no flags specific to output lines, +``GPIOHANDLE_REQUEST_OUTPUT``, ``GPIOHANDLE_REQUEST_OPEN_DRAIN``, or +``GPIOHANDLE_REQUEST_OPEN_SOURCE``, may be set. + +Only one bias flag, ``GPIOHANDLE_REQUEST_BIAS_xxx``, may be set. +If no bias flags are set then the bias configuration is not changed. + +The edge flags, ``GPIOEVENT_REQUEST_RISING_EDGE`` and +``GPIOEVENT_REQUEST_FALLING_EDGE``, may be combined to detect both rising +and falling edges. + +Requesting an invalid configuration is an error (**EINVAL**). + +Return Value +============ + +On success 0 and the :c:type:`request.fd` contains the file +descriptor for the request. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-get-linehandle-ioctl.rst b/Documentation/userspace-api/gpio/gpio-get-linehandle-ioctl.rst new file mode 100644 index 000000000000..9112a9d31174 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-get-linehandle-ioctl.rst @@ -0,0 +1,125 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_GET_LINEHANDLE_IOCTL: + +************************* +GPIO_GET_LINEHANDLE_IOCTL +************************* + +.. warning:: + This ioctl is part of chardev_v1.rst and is obsoleted by + gpio-v2-get-line-ioctl.rst. + +Name +==== + +GPIO_GET_LINEHANDLE_IOCTL - Request a line or lines from the kernel. + +Synopsis +======== + +.. c:macro:: GPIO_GET_LINEHANDLE_IOCTL + +``int ioctl(int chip_fd, GPIO_GET_LINEHANDLE_IOCTL, struct gpiohandle_request *request)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``request`` + The :c:type:`handle_request` specifying the lines to + request and their configuration. + +Description +=========== + +Request a line or lines from the kernel. + +While multiple lines may be requested, the same configuration applies to all +lines in the request. + +On success, the requesting process is granted exclusive access to the line +value and write access to the line configuration. + +The state of a line, including the value of output lines, is guaranteed to +remain as requested until the returned file descriptor is closed. Once the +file descriptor is closed, the state of the line becomes uncontrolled from +the userspace perspective, and may revert to its default state. + +Requesting a line already in use is an error (**EBUSY**). + +Closing the ``chip_fd`` has no effect on existing line handles. + +.. _gpio-get-linehandle-config-rules: + +Configuration Rules +------------------- + +The following configuration rules apply: + +The direction flags, ``GPIOHANDLE_REQUEST_INPUT`` and +``GPIOHANDLE_REQUEST_OUTPUT``, cannot be combined. If neither are set then the +only other flag that may be set is ``GPIOHANDLE_REQUEST_ACTIVE_LOW`` and the +line is requested "as-is" to allow reading of the line value without altering +the electrical configuration. + +The drive flags, ``GPIOHANDLE_REQUEST_OPEN_xxx``, require the +``GPIOHANDLE_REQUEST_OUTPUT`` to be set. +Only one drive flag may be set. +If none are set then the line is assumed push-pull. + +Only one bias flag, ``GPIOHANDLE_REQUEST_BIAS_xxx``, may be set, and +it requires a direction flag to also be set. +If no bias flags are set then the bias configuration is not changed. + +Requesting an invalid configuration is an error (**EINVAL**). + + +.. _gpio-get-linehandle-config-support: + +Configuration Support +--------------------- + +Where the requested configuration is not directly supported by the underlying +hardware and driver, the kernel applies one of these approaches: + + - reject the request + - emulate the feature in software + - treat the feature as best effort + +The approach applied depends on whether the feature can reasonably be emulated +in software, and the impact on the hardware and userspace if the feature is not +supported. +The approach applied for each feature is as follows: + +============== =========== +Feature Approach +============== =========== +Bias best effort +Direction reject +Drive emulate +============== =========== + +Bias is treated as best effort to allow userspace to apply the same +configuration for platforms that support internal bias as those that require +external bias. +Worst case the line floats rather than being biased as expected. + +Drive is emulated by switching the line to an input when the line should not +be driven. + +In all cases, the configuration reported by gpio-get-lineinfo-ioctl.rst +is the requested configuration, not the resulting hardware configuration. +Userspace cannot determine if a feature is supported in hardware, is +emulated, or is best effort. + +Return Value +============ + +On success 0 and the :c:type:`request.fd` contains the +file descriptor for the request. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-get-lineinfo-ioctl.rst b/Documentation/userspace-api/gpio/gpio-get-lineinfo-ioctl.rst new file mode 100644 index 000000000000..c895b8910b25 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-get-lineinfo-ioctl.rst @@ -0,0 +1,54 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_GET_LINEINFO_IOCTL: + +*********************** +GPIO_GET_LINEINFO_IOCTL +*********************** + +.. warning:: + This ioctl is part of chardev_v1.rst and is obsoleted by + gpio-v2-get-lineinfo-ioctl.rst. + +Name +==== + +GPIO_GET_LINEINFO_IOCTL - Get the publicly available information for a line. + +Synopsis +======== + +.. c:macro:: GPIO_GET_LINEINFO_IOCTL + +``int ioctl(int chip_fd, GPIO_GET_LINEINFO_IOCTL, struct gpioline_info *info)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``info`` + The :c:type:`line_info` to be populated, with the + ``offset`` field set to indicate the line to be collected. + +Description +=========== + +Get the publicly available information for a line. + +This information is available independent of whether the line is in use. + +.. note:: + The line info does not include the line value. + + The line must be requested using gpio-get-linehandle-ioctl.rst or + gpio-get-lineevent-ioctl.rst to access its value. + +Return Value +============ + +On success 0 and ``info`` is populated with the chip info. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-get-lineinfo-unwatch-ioctl.rst b/Documentation/userspace-api/gpio/gpio-get-lineinfo-unwatch-ioctl.rst new file mode 100644 index 000000000000..a82d0161daf8 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-get-lineinfo-unwatch-ioctl.rst @@ -0,0 +1,49 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_GET_LINEINFO_UNWATCH_IOCTL: + +******************************* +GPIO_GET_LINEINFO_UNWATCH_IOCTL +******************************* + +Name +==== + +GPIO_GET_LINEINFO_UNWATCH_IOCTL - Disable watching a line for changes to its +requested state and configuration information. + +Synopsis +======== + +.. c:macro:: GPIO_GET_LINEINFO_UNWATCH_IOCTL + +``int ioctl(int chip_fd, GPIO_GET_LINEINFO_UNWATCH_IOCTL, u32 *offset)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``offset`` + The offset of the line to no longer watch. + +Description +=========== + +Remove the line from the list of lines being watched on this ``chip_fd``. + +This is the reverse of gpio-v2-get-lineinfo-watch-ioctl.rst (v2) and +gpio-get-lineinfo-watch-ioctl.rst (v1). + +Unwatching a line that is not watched is an error (**EBUSY**). + +First added in 5.7. + +Return Value +============ + +On success 0. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-get-lineinfo-watch-ioctl.rst b/Documentation/userspace-api/gpio/gpio-get-lineinfo-watch-ioctl.rst new file mode 100644 index 000000000000..f5c92b69a496 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-get-lineinfo-watch-ioctl.rst @@ -0,0 +1,74 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_GET_LINEINFO_WATCH_IOCTL: + +***************************** +GPIO_GET_LINEINFO_WATCH_IOCTL +***************************** + +.. warning:: + This ioctl is part of chardev_v1.rst and is obsoleted by + gpio-v2-get-lineinfo-watch-ioctl.rst. + +Name +==== + +GPIO_GET_LINEINFO_WATCH_IOCTL - Enable watching a line for changes to its +request state and configuration information. + +Synopsis +======== + +.. c:macro:: GPIO_GET_LINEINFO_WATCH_IOCTL + +``int ioctl(int chip_fd, GPIO_GET_LINEINFO_WATCH_IOCTL, struct gpioline_info *info)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``info`` + The :c:type:`line_info` struct to be populated, with + the ``offset`` set to indicate the line to watch + +Description +=========== + +Enable watching a line for changes to its request state and configuration +information. Changes to line info include a line being requested, released +or reconfigured. + +.. note:: + Watching line info is not generally required, and would typically only be + used by a system monitoring component. + + The line info does NOT include the line value. + + The line must be requested using gpio-get-linehandle-ioctl.rst or + gpio-get-lineevent-ioctl.rst to access its value, and the line event can + monitor a line for events using gpio-lineevent-data-read.rst. + +By default all lines are unwatched when the GPIO chip is opened. + +Multiple lines may be watched simultaneously by adding a watch for each. + +Once a watch is set, any changes to line info will generate events which can be +read from the ``chip_fd`` as described in +gpio-lineinfo-changed-read.rst. + +Adding a watch to a line that is already watched is an error (**EBUSY**). + +Watches are specific to the ``chip_fd`` and are independent of watches +on the same GPIO chip opened with a separate call to `open()`. + +First added in 5.7. + +Return Value +============ + +On success 0 and ``info`` is populated with the current line info. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-handle-get-line-values-ioctl.rst b/Documentation/userspace-api/gpio/gpio-handle-get-line-values-ioctl.rst new file mode 100644 index 000000000000..25263b8f0588 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-handle-get-line-values-ioctl.rst @@ -0,0 +1,56 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIOHANDLE_GET_LINE_VALUES_IOCTL: + +******************************** +GPIOHANDLE_GET_LINE_VALUES_IOCTL +******************************** +.. warning:: + This ioctl is part of chardev_v1.rst and is obsoleted by + gpio-v2-line-get-values-ioctl.rst. + +Name +==== + +GPIOHANDLE_GET_LINE_VALUES_IOCTL - Get the values of all requested lines. + +Synopsis +======== + +.. c:macro:: GPIOHANDLE_GET_LINE_VALUES_IOCTL + +``int ioctl(int handle_fd, GPIOHANDLE_GET_LINE_VALUES_IOCTL, struct gpiohandle_data *values)`` + +Arguments +========= + +``handle_fd`` + The file descriptor of the GPIO character device, as returned in the + :c:type:`request.fd` by gpio-get-linehandle-ioctl.rst. + +``values`` + The :c:type:`line_values` to be populated. + +Description +=========== + +Get the values of all requested lines. + +The values of both input and output lines may be read. + +For output lines, the value returned is driver and configuration dependent and +may be either the output buffer (the last requested value set) or the input +buffer (the actual level of the line), and depending on the hardware and +configuration these may differ. + +This ioctl can also be used to read the line value for line events, +substituting the ``event_fd`` for the ``handle_fd``. As there is only +one line requested in that case, only the one value is returned in ``values``. + +Return Value +============ + +On success 0 and ``values`` populated with the values read. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-handle-set-config-ioctl.rst b/Documentation/userspace-api/gpio/gpio-handle-set-config-ioctl.rst new file mode 100644 index 000000000000..d002a84681ac --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-handle-set-config-ioctl.rst @@ -0,0 +1,63 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIOHANDLE_SET_CONFIG_IOCTL: + +*************************** +GPIOHANDLE_SET_CONFIG_IOCTL +*************************** + +.. warning:: + This ioctl is part of chardev_v1.rst and is obsoleted by + gpio-v2-line-set-config-ioctl.rst. + +Name +==== + +GPIOHANDLE_SET_CONFIG_IOCTL - Update the configuration of previously requested lines. + +Synopsis +======== + +.. c:macro:: GPIOHANDLE_SET_CONFIG_IOCTL + +``int ioctl(int handle_fd, GPIOHANDLE_SET_CONFIG_IOCTL, struct gpiohandle_config *config)`` + +Arguments +========= + +``handle_fd`` + The file descriptor of the GPIO character device, as returned in the + :c:type:`request.fd` by gpio-get-linehandle-ioctl.rst. + +``config`` + The new :c:type:`configuration` to apply to the + requested lines. + +Description +=========== + +Update the configuration of previously requested lines, without releasing the +line or introducing potential glitches. + +The configuration applies to all requested lines. + +The same :ref:`gpio-get-linehandle-config-rules` and +:ref:`gpio-get-linehandle-config-support` that apply when requesting the +lines also apply when updating the line configuration. + +The motivating use case for this command is changing direction of +bi-directional lines between input and output, but it may be used more +generally to move lines seamlessly from one configuration state to another. + +To only change the value of output lines, use +gpio-handle-set-line-values-ioctl.rst. + +First added in 5.5. + +Return Value +============ + +On success 0. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-handle-set-line-values-ioctl.rst b/Documentation/userspace-api/gpio/gpio-handle-set-line-values-ioctl.rst new file mode 100644 index 000000000000..0aa05e623a6c --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-handle-set-line-values-ioctl.rst @@ -0,0 +1,48 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_HANDLE_SET_LINE_VALUES_IOCTL: + +********************************* +GPIO_HANDLE_SET_LINE_VALUES_IOCTL +********************************* +.. warning:: + This ioctl is part of chardev_v1.rst and is obsoleted by + gpio-v2-line-set-values-ioctl.rst. + +Name +==== + +GPIO_HANDLE_SET_LINE_VALUES_IOCTL - Set the values of all requested output lines. + +Synopsis +======== + +.. c:macro:: GPIO_HANDLE_SET_LINE_VALUES_IOCTL + +``int ioctl(int handle_fd, GPIO_HANDLE_SET_LINE_VALUES_IOCTL, struct gpiohandle_data *values)`` + +Arguments +========= + +``handle_fd`` + The file descriptor of the GPIO character device, as returned in the + :c:type:`request.fd` by gpio-get-linehandle-ioctl.rst. + +``values`` + The :c:type:`line_values` to set. + +Description +=========== + +Set the values of all requested output lines. + +Only the values of output lines may be set. +Attempting to set the value of input lines is an error (**EPERM**). + +Return Value +============ + +On success 0. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-lineevent-data-read.rst b/Documentation/userspace-api/gpio/gpio-lineevent-data-read.rst new file mode 100644 index 000000000000..68b8d4f9f604 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-lineevent-data-read.rst @@ -0,0 +1,84 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_LINEEVENT_DATA_READ: + +************************ +GPIO_LINEEVENT_DATA_READ +************************ + +.. warning:: + This ioctl is part of chardev_v1.rst and is obsoleted by + gpio-v2-line-event-read.rst. + +Name +==== + +GPIO_LINEEVENT_DATA_READ - Read edge detection events from a line event. + +Synopsis +======== + +``int read(int event_fd, void *buf, size_t count)`` + +Arguments +========= + +``event_fd`` + The file descriptor of the GPIO character device, as returned in the + :c:type:`request.fd` by gpio-get-lineevent-ioctl.rst. + +``buf`` + The buffer to contain the :c:type:`events`. + +``count`` + The number of bytes available in ``buf``, which must be at + least the size of a :c:type:`gpioevent_data`. + +Description +=========== + +Read edge detection events for a line from a line event. + +Edge detection must be enabled for the input line using either +``GPIOEVENT_REQUEST_RISING_EDGE`` or ``GPIOEVENT_REQUEST_FALLING_EDGE``, or +both. Edge events are then generated whenever edge interrupts are detected on +the input line. + +The kernel captures and timestamps edge events as close as possible to their +occurrence and stores them in a buffer from where they can be read by +userspace at its convenience using `read()`. + +The source of the clock for :c:type:`event.timestamp` is +``CLOCK_MONOTONIC``, except for kernels earlier than Linux 5.7 when it was +``CLOCK_REALTIME``. There is no indication in the :c:type:`gpioevent_data` +as to which clock source is used, it must be determined from either the kernel +version or sanity checks on the timestamp itself. + +Events read from the buffer are always in the same order that they were +detected by the kernel. + +The size of the kernel event buffer is fixed at 16 events. + +The buffer may overflow if bursts of events occur quicker than they are read +by userspace. If an overflow occurs then the most recent event is discarded. +Overflow cannot be detected from userspace. + +To minimize the number of calls required to copy events from the kernel to +userspace, `read()` supports copying multiple events. The number of events +copied is the lower of the number available in the kernel buffer and the +number that will fit in the userspace buffer (``buf``). + +The `read()` will block if no event is available and the ``event_fd`` has not +been set **O_NONBLOCK**. + +The presence of an event can be tested for by checking that the ``event_fd`` is +readable using `poll()` or an equivalent. + +Return Value +============ + +On success the number of bytes read, which will be a multiple of the size of +a :c:type:`gpio_lineevent_data` event. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-lineinfo-changed-read.rst b/Documentation/userspace-api/gpio/gpio-lineinfo-changed-read.rst new file mode 100644 index 000000000000..c4f5e1787a9d --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-lineinfo-changed-read.rst @@ -0,0 +1,87 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_LINEINFO_CHANGED_READ: + +************************** +GPIO_LINEINFO_CHANGED_READ +************************** + +.. warning:: + This ioctl is part of chardev_v1.rst and is obsoleted by + gpio-v2-lineinfo-changed-read.rst. + +Name +==== + +GPIO_LINEINFO_CHANGED_READ - Read line info change events for watched lines +from the chip. + +Synopsis +======== + +``int read(int chip_fd, void *buf, size_t count)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``buf`` + The buffer to contain the :c:type:`events`. + +``count`` + The number of bytes available in ``buf``, which must be at least the size + of a :c:type:`gpioline_info_changed` event. + +Description +=========== + +Read line info change events for watched lines from the chip. + +.. note:: + Monitoring line info changes is not generally required, and would typically + only be performed by a system monitoring component. + + These events relate to changes in a line's request state or configuration, + not its value. Use gpio-lineevent-data-read.rst to receive events when a + line changes value. + +A line must be watched using gpio-get-lineinfo-watch-ioctl.rst to generate +info changed events. Subsequently, a request, release, or reconfiguration +of the line will generate an info changed event. + +The kernel timestamps events when they occur and stores them in a buffer +from where they can be read by userspace at its convenience using `read()`. + +The size of the kernel event buffer is fixed at 32 events per ``chip_fd``. + +The buffer may overflow if bursts of events occur quicker than they are read +by userspace. If an overflow occurs then the most recent event is discarded. +Overflow cannot be detected from userspace. + +Events read from the buffer are always in the same order that they were +detected by the kernel, including when multiple lines are being monitored by +the one ``chip_fd``. + +To minimize the number of calls required to copy events from the kernel to +userspace, `read()` supports copying multiple events. The number of events +copied is the lower of the number available in the kernel buffer and the +number that will fit in the userspace buffer (``buf``). + +A `read()` will block if no event is available and the ``chip_fd`` has not +been set **O_NONBLOCK**. + +The presence of an event can be tested for by checking that the ``chip_fd`` is +readable using `poll()` or an equivalent. + +First added in 5.7. + +Return Value +============ + +On success the number of bytes read, which will be a multiple of the size of +a :c:type:`gpioline_info_changed` event. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-v2-get-line-ioctl.rst b/Documentation/userspace-api/gpio/gpio-v2-get-line-ioctl.rst new file mode 100644 index 000000000000..56b975801b6a --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-v2-get-line-ioctl.rst @@ -0,0 +1,152 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_V2_GET_LINE_IOCTL: + +********************** +GPIO_V2_GET_LINE_IOCTL +********************** + +Name +==== + +GPIO_V2_GET_LINE_IOCTL - Request a line or lines from the kernel. + +Synopsis +======== + +.. c:macro:: GPIO_V2_GET_LINE_IOCTL + +``int ioctl(int chip_fd, GPIO_V2_GET_LINE_IOCTL, struct gpio_v2_line_request *request)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``request`` + The :c:type:`line_request` specifying the lines + to request and their configuration. + +Description +=========== + +On success, the requesting process is granted exclusive access to the line +value, write access to the line configuration, and may receive events when +edges are detected on the line, all of which are described in more detail in +:ref:`gpio-v2-line-request`. + +A number of lines may be requested in the one line request, and request +operations are performed on the requested lines by the kernel as atomically +as possible. e.g. gpio-v2-line-get-values-ioctl.rst will read all the +requested lines at once. + +The state of a line, including the value of output lines, is guaranteed to +remain as requested until the returned file descriptor is closed. Once the +file descriptor is closed, the state of the line becomes uncontrolled from +the userspace perspective, and may revert to its default state. + +Requesting a line already in use is an error (**EBUSY**). + +Closing the ``chip_fd`` has no effect on existing line requests. + +.. _gpio-v2-get-line-config-rules: + +Configuration Rules +------------------- + +For any given requested line, the following configuration rules apply: + +The direction flags, ``GPIO_V2_LINE_FLAG_INPUT`` and +``GPIO_V2_LINE_FLAG_OUTPUT``, cannot be combined. If neither are set then +the only other flag that may be set is ``GPIO_V2_LINE_FLAG_ACTIVE_LOW`` +and the line is requested "as-is" to allow reading of the line value +without altering the electrical configuration. + +The drive flags, ``GPIO_V2_LINE_FLAG_OPEN_xxx``, require the +``GPIO_V2_LINE_FLAG_OUTPUT`` to be set. +Only one drive flag may be set. +If none are set then the line is assumed push-pull. + +Only one bias flag, ``GPIO_V2_LINE_FLAG_BIAS_xxx``, may be set, and it +requires a direction flag to also be set. +If no bias flags are set then the bias configuration is not changed. + +The edge flags, ``GPIO_V2_LINE_FLAG_EDGE_xxx``, require +``GPIO_V2_LINE_FLAG_INPUT`` to be set and may be combined to detect both rising +and falling edges. Requesting edge detection from a line that does not support +it is an error (**ENXIO**). + +Only one event clock flag, ``GPIO_V2_LINE_FLAG_EVENT_CLOCK_xxx``, may be set. +If none are set then the event clock defaults to ``CLOCK_MONOTONIC``. +The ``GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE`` flag requires supporting hardware +and a kernel with ``CONFIG_HTE`` set. Requesting HTE from a device that +doesn't support it is an error (**EOPNOTSUP**). + +The :c:type:`debounce_period_us` attribute may only +be applied to lines with ``GPIO_V2_LINE_FLAG_INPUT`` set. When set, debounce +applies to both the values returned by gpio-v2-line-get-values-ioctl.rst and +the edges returned by gpio-v2-line-event-read.rst. If not +supported directly by hardware, debouncing is emulated in software by the +kernel. Requesting debounce on a line that supports neither debounce in +hardware nor interrupts, as required for software emulation, is an error +(**ENXIO**). + +Requesting an invalid configuration is an error (**EINVAL**). + +.. _gpio-v2-get-line-config-support: + +Configuration Support +--------------------- + +Where the requested configuration is not directly supported by the underlying +hardware and driver, the kernel applies one of these approaches: + + - reject the request + - emulate the feature in software + - treat the feature as best effort + +The approach applied depends on whether the feature can reasonably be emulated +in software, and the impact on the hardware and userspace if the feature is not +supported. +The approach applied for each feature is as follows: + +============== =========== +Feature Approach +============== =========== +Bias best effort +Debounce emulate +Direction reject +Drive emulate +Edge Detection reject +============== =========== + +Bias is treated as best effort to allow userspace to apply the same +configuration for platforms that support internal bias as those that require +external bias. +Worst case the line floats rather than being biased as expected. + +Debounce is emulated by applying a filter to hardware interrupts on the line. +An edge event is generated after an edge is detected and the line remains +stable for the debounce period. +The event timestamp corresponds to the end of the debounce period. + +Drive is emulated by switching the line to an input when the line should not +be actively driven. + +Edge detection requires interrupt support, and is rejected if that is not +supported. Emulation by polling can still be performed from userspace. + +In all cases, the configuration reported by gpio-v2-get-lineinfo-ioctl.rst +is the requested configuration, not the resulting hardware configuration. +Userspace cannot determine if a feature is supported in hardware, is +emulated, or is best effort. + +Return Value +============ + +On success 0 and the :c:type:`request.fd` contains the +file descriptor for the request. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-v2-get-lineinfo-ioctl.rst b/Documentation/userspace-api/gpio/gpio-v2-get-lineinfo-ioctl.rst new file mode 100644 index 000000000000..bc4d8df887d4 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-v2-get-lineinfo-ioctl.rst @@ -0,0 +1,50 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_V2_GET_LINEINFO_IOCTL: + +************************** +GPIO_V2_GET_LINEINFO_IOCTL +************************** + +Name +==== + +GPIO_V2_GET_LINEINFO_IOCTL - Get the publicly available information for a line. + +Synopsis +======== + +.. c:macro:: GPIO_V2_GET_LINEINFO_IOCTL + +``int ioctl(int chip_fd, GPIO_V2_GET_LINEINFO_IOCTL, struct gpio_v2_line_info *info)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``info`` + The :c:type:`line_info` to be populated, with the + ``offset`` field set to indicate the line to be collected. + +Description +=========== + +Get the publicly available information for a line. + +This information is available independent of whether the line is in use. + +.. note:: + The line info does not include the line value. + + The line must be requested using gpio-v2-get-line-ioctl.rst to access its + value. + +Return Value +============ + +On success 0 and ``info`` is populated with the chip info. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-v2-get-lineinfo-watch-ioctl.rst b/Documentation/userspace-api/gpio/gpio-v2-get-lineinfo-watch-ioctl.rst new file mode 100644 index 000000000000..938ff85a9322 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-v2-get-lineinfo-watch-ioctl.rst @@ -0,0 +1,67 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_V2_GET_LINEINFO_WATCH_IOCTL: + +******************************** +GPIO_V2_GET_LINEINFO_WATCH_IOCTL +******************************** + +Name +==== + +GPIO_V2_GET_LINEINFO_WATCH_IOCTL - Enable watching a line for changes to its +request state and configuration information. + +Synopsis +======== + +.. c:macro:: GPIO_V2_GET_LINEINFO_WATCH_IOCTL + +``int ioctl(int chip_fd, GPIO_V2_GET_LINEINFO_WATCH_IOCTL, struct gpio_v2_line_info *info)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``info`` + The :c:type:`line_info` struct to be populated, with + the ``offset`` set to indicate the line to watch + +Description +=========== + +Enable watching a line for changes to its request state and configuration +information. Changes to line info include a line being requested, released +or reconfigured. + +.. note:: + Watching line info is not generally required, and would typically only be + used by a system monitoring component. + + The line info does NOT include the line value. + The line must be requested using gpio-v2-get-line-ioctl.rst to access + its value, and the line request can monitor a line for events using + gpio-v2-line-event-read.rst. + +By default all lines are unwatched when the GPIO chip is opened. + +Multiple lines may be watched simultaneously by adding a watch for each. + +Once a watch is set, any changes to line info will generate events which can be +read from the ``chip_fd`` as described in +gpio-v2-lineinfo-changed-read.rst. + +Adding a watch to a line that is already watched is an error (**EBUSY**). + +Watches are specific to the ``chip_fd`` and are independent of watches +on the same GPIO chip opened with a separate call to `open()`. + +Return Value +============ + +On success 0 and ``info`` is populated with the current line info. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-v2-line-event-read.rst b/Documentation/userspace-api/gpio/gpio-v2-line-event-read.rst new file mode 100644 index 000000000000..6513c23fb7ca --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-v2-line-event-read.rst @@ -0,0 +1,83 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_V2_LINE_EVENT_READ: + +*********************** +GPIO_V2_LINE_EVENT_READ +*********************** + +Name +==== + +GPIO_V2_LINE_EVENT_READ - Read edge detection events for lines from a request. + +Synopsis +======== + +``int read(int req_fd, void *buf, size_t count)`` + +Arguments +========= + +``req_fd`` + The file descriptor of the GPIO character device, as returned in the + :c:type:`request.fd` by gpio-v2-get-line-ioctl.rst. + +``buf`` + The buffer to contain the :c:type:`events`. + +``count`` + The number of bytes available in ``buf``, which must be at + least the size of a :c:type:`gpio_v2_line_event`. + +Description +=========== + +Read edge detection events for lines from a request. + +Edge detection must be enabled for the input line using either +``GPIO_V2_LINE_FLAG_EDGE_RISING`` or ``GPIO_V2_LINE_FLAG_EDGE_FALLING``, or +both. Edge events are then generated whenever edge interrupts are detected on +the input line. + +The kernel captures and timestamps edge events as close as possible to their +occurrence and stores them in a buffer from where they can be read by +userspace at its convenience using `read()`. + +Events read from the buffer are always in the same order that they were +detected by the kernel, including when multiple lines are being monitored by +the one request. + +The size of the kernel event buffer is fixed at the time of line request +creation, and can be influenced by the +:c:type:`request.event_buffer_size`. +The default size is 16 times the number of lines requested. + +The buffer may overflow if bursts of events occur quicker than they are read +by userspace. If an overflow occurs then the oldest buffered event is +discarded. Overflow can be detected from userspace by monitoring the event +sequence numbers. + +To minimize the number of calls required to copy events from the kernel to +userspace, `read()` supports copying multiple events. The number of events +copied is the lower of the number available in the kernel buffer and the +number that will fit in the userspace buffer (``buf``). + +Changing the edge detection flags using gpio-v2-line-set-config-ioctl.rst +does not remove or modify the events already contained in the kernel event +buffer. + +The `read()` will block if no event is available and the ``req_fd`` has not +been set **O_NONBLOCK**. + +The presence of an event can be tested for by checking that the ``req_fd`` is +readable using `poll()` or an equivalent. + +Return Value +============ + +On success the number of bytes read, which will be a multiple of the size of a +:c:type:`gpio_v2_line_event` event. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-v2-line-get-values-ioctl.rst b/Documentation/userspace-api/gpio/gpio-v2-line-get-values-ioctl.rst new file mode 100644 index 000000000000..e4e74a1926d8 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-v2-line-get-values-ioctl.rst @@ -0,0 +1,51 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_V2_LINE_GET_VALUES_IOCTL: + +***************************** +GPIO_V2_LINE_GET_VALUES_IOCTL +***************************** + +Name +==== + +GPIO_V2_LINE_GET_VALUES_IOCTL - Get the values of requested lines. + +Synopsis +======== + +.. c:macro:: GPIO_V2_LINE_GET_VALUES_IOCTL + +``int ioctl(int req_fd, GPIO_V2_LINE_GET_VALUES_IOCTL, struct gpio_v2_line_values *values)`` + +Arguments +========= + +``req_fd`` + The file descriptor of the GPIO character device, as returned in the + :c:type:`request.fd` by gpio-v2-get-line-ioctl.rst. + +``values`` + The :c:type:`line_values` to get with the ``mask`` set + to indicate the subset of requested lines to get. + +Description +=========== + +Get the values of requested lines. + +The values of both input and output lines may be read. + +For output lines, the value returned is driver and configuration dependent and +may be either the output buffer (the last requested value set) or the input +buffer (the actual level of the line), and depending on the hardware and +configuration these may differ. + +Return Value +============ + +On success 0 and the corresponding :c:type:`values.bits` +contain the value read. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-v2-line-set-config-ioctl.rst b/Documentation/userspace-api/gpio/gpio-v2-line-set-config-ioctl.rst new file mode 100644 index 000000000000..9b942a8a53ca --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-v2-line-set-config-ioctl.rst @@ -0,0 +1,58 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_V2_LINE_SET_CONFIG_IOCTL: + +***************************** +GPIO_V2_LINE_SET_CONFIG_IOCTL +***************************** + +Name +==== + +GPIO_V2_LINE_SET_CONFIG_IOCTL - Update the configuration of previously requested lines. + +Synopsis +======== + +.. c:macro:: GPIO_V2_LINE_SET_CONFIG_IOCTL + +``int ioctl(int req_fd, GPIO_V2_LINE_SET_CONFIG_IOCTL, struct gpio_v2_line_config *config)`` + +Arguments +========= + +``req_fd`` + The file descriptor of the GPIO character device, as returned in the + :c:type:`request.fd` by gpio-v2-get-line-ioctl.rst. + +``config`` + The new :c:type:`configuration` to apply to the + requested lines. + +Description +=========== + +Update the configuration of previously requested lines, without releasing the +line or introducing potential glitches. + +The new configuration must specify the configuration of all requested lines. + +The same :ref:`gpio-v2-get-line-config-rules` and +:ref:`gpio-v2-get-line-config-support` that apply when requesting the lines +also apply when updating the line configuration. + +The motivating use case for this command is changing direction of +bi-directional lines between input and output, but it may also be used to +dynamically control edge detection, or more generally move lines seamlessly +from one configuration state to another. + +To only change the value of output lines, use +gpio-v2-line-set-values-ioctl.rst. + +Return Value +============ + +On success 0. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-v2-line-set-values-ioctl.rst b/Documentation/userspace-api/gpio/gpio-v2-line-set-values-ioctl.rst new file mode 100644 index 000000000000..6d2d1886950b --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-v2-line-set-values-ioctl.rst @@ -0,0 +1,47 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_V2_LINE_SET_VALUES_IOCTL: + +***************************** +GPIO_V2_LINE_SET_VALUES_IOCTL +***************************** + +Name +==== + +GPIO_V2_LINE_SET_VALUES_IOCTL - Set the values of requested output lines. + +Synopsis +======== + +.. c:macro:: GPIO_V2_LINE_SET_VALUES_IOCTL + +``int ioctl(int req_fd, GPIO_V2_LINE_SET_VALUES_IOCTL, struct gpio_v2_line_values *values)`` + +Arguments +========= + +``req_fd`` + The file descriptor of the GPIO character device, as returned in the + :c:type:`request.fd` by gpio-v2-get-line-ioctl.rst. + +``values`` + The :c:type:`line_values` to set with the ``mask`` set + to indicate the subset of requested lines to set and ``bits`` set to + indicate the new value. + +Description +=========== + +Set the values of requested output lines. + +Only the values of output lines may be set. +Attempting to set the value of an input line is an error (**EPERM**). + +Return Value +============ + +On success 0. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/gpio-v2-lineinfo-changed-read.rst b/Documentation/userspace-api/gpio/gpio-v2-lineinfo-changed-read.rst new file mode 100644 index 000000000000..24ad325e7253 --- /dev/null +++ b/Documentation/userspace-api/gpio/gpio-v2-lineinfo-changed-read.rst @@ -0,0 +1,81 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. _GPIO_V2_LINEINFO_CHANGED_READ: + +***************************** +GPIO_V2_LINEINFO_CHANGED_READ +***************************** + +Name +==== + +GPIO_V2_LINEINFO_CHANGED_READ - Read line info changed events for watched +lines from the chip. + +Synopsis +======== + +``int read(int chip_fd, void *buf, size_t count)`` + +Arguments +========= + +``chip_fd`` + The file descriptor of the GPIO character device returned by `open()`. + +``buf`` + The buffer to contain the :c:type:`events`. + +``count`` + The number of bytes available in ``buf``, which must be at least the size + of a :c:type:`gpio_v2_line_info_changed` event. + +Description +=========== + +Read line info changed events for watched lines from the chip. + +.. note:: + Monitoring line info changes is not generally required, and would typically + only be performed by a system monitoring component. + + These events relate to changes in a line's request state or configuration, + not its value. Use gpio-v2-line-event-read.rst to receive events when a + line changes value. + +A line must be watched using gpio-v2-get-lineinfo-watch-ioctl.rst to generate +info changed events. Subsequently, a request, release, or reconfiguration +of the line will generate an info changed event. + +The kernel timestamps events when they occur and stores them in a buffer +from where they can be read by userspace at its convenience using `read()`. + +The size of the kernel event buffer is fixed at 32 events per ``chip_fd``. + +The buffer may overflow if bursts of events occur quicker than they are read +by userspace. If an overflow occurs then the most recent event is discarded. +Overflow cannot be detected from userspace. + +Events read from the buffer are always in the same order that they were +detected by the kernel, including when multiple lines are being monitored by +the one ``chip_fd``. + +To minimize the number of calls required to copy events from the kernel to +userspace, `read()` supports copying multiple events. The number of events +copied is the lower of the number available in the kernel buffer and the +number that will fit in the userspace buffer (``buf``). + +A `read()` will block if no event is available and the ``chip_fd`` has not +been set **O_NONBLOCK**. + +The presence of an event can be tested for by checking that the ``chip_fd`` is +readable using `poll()` or an equivalent. + +Return Value +============ + +On success the number of bytes read, which will be a multiple of the size +of a :c:type:`gpio_v2_line_info_changed` event. + +On error -1 and the ``errno`` variable is set appropriately. +Common error codes are described in error-codes.rst. diff --git a/Documentation/userspace-api/gpio/index.rst b/Documentation/userspace-api/gpio/index.rst new file mode 100644 index 000000000000..f258de4ef370 --- /dev/null +++ b/Documentation/userspace-api/gpio/index.rst @@ -0,0 +1,18 @@ +.. SPDX-License-Identifier: GPL-2.0 + +==== +GPIO +==== + +.. toctree:: + :maxdepth: 1 + + Character Device Userspace API + Obsolete Userspace APIs + +.. only:: subproject and html + + Indices + ======= + + * :ref:`genindex` diff --git a/Documentation/userspace-api/gpio/obsolete.rst b/Documentation/userspace-api/gpio/obsolete.rst new file mode 100644 index 000000000000..c42538b44ec8 --- /dev/null +++ b/Documentation/userspace-api/gpio/obsolete.rst @@ -0,0 +1,11 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============================ +Obsolete GPIO Userspace APIs +============================ + +.. toctree:: + :maxdepth: 1 + + Character Device Userspace API (v1) + Sysfs Interface diff --git a/Documentation/admin-guide/gpio/sysfs.rst b/Documentation/userspace-api/gpio/sysfs.rst similarity index 87% rename from Documentation/admin-guide/gpio/sysfs.rst rename to Documentation/userspace-api/gpio/sysfs.rst index 35171d15f78d..116921048b18 100644 --- a/Documentation/admin-guide/gpio/sysfs.rst +++ b/Documentation/userspace-api/gpio/sysfs.rst @@ -2,18 +2,18 @@ GPIO Sysfs Interface for Userspace ================================== .. warning:: + This API is obsoleted by the chardev.rst and the ABI documentation has + been moved to Documentation/ABI/obsolete/sysfs-gpio. - THIS ABI IS DEPRECATED, THE ABI DOCUMENTATION HAS BEEN MOVED TO - Documentation/ABI/obsolete/sysfs-gpio AND NEW USERSPACE CONSUMERS - ARE SUPPOSED TO USE THE CHARACTER DEVICE ABI. THIS OLD SYSFS ABI WILL - NOT BE DEVELOPED (NO NEW FEATURES), IT WILL JUST BE MAINTAINED. + New developments should use the chardev.rst, and existing developments are + encouraged to migrate as soon as possible, as this API will be removed + in the future. -Refer to the examples in tools/gpio/* for an introduction to the new -character device ABI. Also see the userspace header in -include/uapi/linux/gpio.h + This interface will continue to be maintained for the migration period, + but new features will only be added to the new API. -The deprecated sysfs ABI ------------------------- +The obsolete sysfs ABI +---------------------- Platforms which use the "gpiolib" implementors framework may choose to configure a sysfs user interface to GPIOs. This is different from the debugfs interface, since it provides control over GPIO direction and @@ -33,9 +33,12 @@ userspace GPIO can be used to determine system configuration data that standard kernels won't know about. And for some tasks, simple userspace GPIO drivers could be all that the system really needs. -DO NOT ABUSE SYSFS TO CONTROL HARDWARE THAT HAS PROPER KERNEL DRIVERS. -PLEASE READ THE DOCUMENT AT Documentation/driver-api/gpio/drivers-on-gpio.rst -TO AVOID REINVENTING KERNEL WHEELS IN USERSPACE. I MEAN IT. REALLY. +.. note:: + Do NOT abuse sysfs to control hardware that has proper kernel drivers. + Please read Documentation/driver-api/gpio/drivers-on-gpio.rst + to avoid reinventing kernel wheels in userspace. + + I MEAN IT. REALLY. Paths in Sysfs -------------- @@ -84,9 +87,9 @@ and have the following read/write attributes: allow userspace to reconfigure this GPIO's direction. "value" ... - reads as either 0 (low) or 1 (high). If the GPIO + reads as either 0 (inactive) or 1 (active). If the GPIO is configured as an output, this value may be written; - any nonzero value is treated as high. + any nonzero value is treated as active. If the pin can be configured as interrupt-generating interrupt and if it has been configured to generate interrupts (see the diff --git a/Documentation/userspace-api/index.rst b/Documentation/userspace-api/index.rst index 67d663cf2ff2..afecfe3cc4a8 100644 --- a/Documentation/userspace-api/index.rst +++ b/Documentation/userspace-api/index.rst @@ -42,6 +42,7 @@ Devices and I/O accelerators/ocxl dma-buf-alloc-exchange + gpio/index iommu iommufd media/index diff --git a/MAINTAINERS b/MAINTAINERS index 477c6a144499..bdfe1ec22f7c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1104,6 +1104,14 @@ F: Documentation/devicetree/bindings/perf/amlogic,g12-ddr-pmu.yaml F: drivers/perf/amlogic/ F: include/soc/amlogic/ +AMPHENOL CHIPCAP 2 HUMIDITY-TEMPERATURE IIO DRIVER +M: Javier Carrasco +L: linux-hwmon@vger.kernel.org +S: Maintained +F: Documentation/devicetree/bindings/hwmon/amphenol,chipcap2.yaml +F: Documentation/hwmon/chipcap2.rst +F: drivers/hwmon/chipcap2.c + AMPHION VPU CODEC V4L2 DRIVER M: Ming Qian M: Zhou Peng @@ -1390,15 +1398,6 @@ F: drivers/iio/amplifiers/hmc425a.c F: drivers/staging/iio/*/ad* X: drivers/iio/*/adjd* -ANALOG DEVICES INC MAX31760 DRIVER -M: Ibrahim Tilki -S: Maintained -W: http://wiki.analog.com/ -W: https://ez.analog.com/linux-software-drivers -F: Documentation/devicetree/bindings/hwmon/adi,max31760.yaml -F: Documentation/hwmon/max31760.rst -F: drivers/hwmon/max31760.c - ANALOGBITS PLL LIBRARIES M: Paul Walmsley M: Samuel Holland @@ -3190,6 +3189,12 @@ S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git F: drivers/platform/x86/asus-tf103c-dock.c +ASUS ROG RYUJIN AIO HARDWARE MONITOR DRIVER +M: Aleksa Savic +L: linux-hwmon@vger.kernel.org +S: Maintained +F: drivers/hwmon/asus_rog_ryujin.c + ASUS WIRELESS RADIO CONTROL DRIVER M: João Paulo Rechi Vita L: platform-driver-x86@vger.kernel.org @@ -6134,6 +6139,14 @@ F: include/linux/device-mapper.h F: include/linux/dm-*.h F: include/uapi/linux/dm-*.h +DEVICE-MAPPER VDO TARGET +M: Matthew Sakai +M: dm-devel@lists.linux.dev +L: dm-devel@lists.linux.dev +S: Maintained +F: Documentation/admin-guide/device-mapper/vdo*.rst +F: drivers/md/dm-vdo/ + DEVLINK M: Jiri Pirko L: netdev@vger.kernel.org @@ -7801,8 +7814,8 @@ F: drivers/media/usb/em28xx/ EMMC CMDQ HOST CONTROLLER INTERFACE (CQHCI) DRIVER M: Adrian Hunter -M: Ritesh Harjani -M: Asutosh Das +M: Asutosh Das +R: Ritesh Harjani L: linux-mmc@vger.kernel.org S: Supported F: drivers/mmc/host/cqhci* @@ -9182,6 +9195,7 @@ S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git F: Documentation/ABI/obsolete/sysfs-gpio F: Documentation/ABI/testing/gpio-cdev +F: Documentation/userspace-api/gpio/ F: drivers/gpio/gpiolib-cdev.c F: include/uapi/linux/gpio.h F: tools/gpio/ @@ -10522,22 +10536,6 @@ L: linux-fbdev@vger.kernel.org S: Orphan F: drivers/video/fbdev/imsttfb.c -INA209 HARDWARE MONITOR DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml -F: Documentation/hwmon/ina209.rst -F: drivers/hwmon/ina209.c - -INA2XX HARDWARE MONITOR DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/hwmon/ina2xx.rst -F: drivers/hwmon/ina2xx.c -F: include/linux/platform_data/ina2xx.h - INCREMENTAL FILE SYSTEM M: Paul Lawrence L: linux-unionfs@vger.kernel.org @@ -11290,7 +11288,6 @@ F: drivers/iommu/ F: include/linux/iommu.h F: include/linux/iova.h F: include/linux/of_iommu.h -F: include/uapi/linux/iommu.h IOMMUFD M: Jason Gunthorpe @@ -11530,14 +11527,6 @@ S: Maintained F: arch/x86/include/asm/jailhouse_para.h F: arch/x86/kernel/jailhouse.c -JC42.4 TEMPERATURE SENSOR DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/devicetree/bindings/hwmon/jedec,jc42.yaml -F: Documentation/hwmon/jc42.rst -F: drivers/hwmon/jc42.c - JFS FILESYSTEM M: Dave Kleikamp L: jfs-discussion@lists.sourceforge.net @@ -12602,13 +12591,6 @@ F: Documentation/hwmon/lm90.rst F: drivers/hwmon/lm90.c F: include/dt-bindings/thermal/lm90.h -LM95234 HARDWARE MONITOR DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/hwmon/lm95234.rst -F: drivers/hwmon/lm95234.c - LME2510 MEDIA DRIVER M: Malcolm Priestley L: linux-media@vger.kernel.org @@ -12812,12 +12794,13 @@ W: https://ez.analog.com/linux-software-drivers F: Documentation/devicetree/bindings/iio/temperature/adi,ltc2983.yaml F: drivers/iio/temperature/ltc2983.c -LTC4261 HARDWARE MONITOR DRIVER -M: Guenter Roeck +LTC4282 HARDWARE MONITOR DRIVER +M: Nuno Sa L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/hwmon/ltc4261.rst -F: drivers/hwmon/ltc4261.c +S: Supported +F: Documentation/devicetree/bindings/hwmon/adi,ltc4282.yaml +F: Documentation/hwmon/ltc4282.rst +F: drivers/hwmon/ltc4282.c LTC4286 HARDWARE MONITOR DRIVER M: Delphine CC Chiu @@ -13175,13 +13158,6 @@ S: Maintained F: Documentation/hwmon/max15301.rst F: drivers/hwmon/pmbus/max15301.c -MAX16065 HARDWARE MONITOR DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/hwmon/max16065.rst -F: drivers/hwmon/max16065.c - MAX2175 SDR TUNER DRIVER M: Ramesh Shanmugasundaram L: linux-media@vger.kernel.org @@ -13192,15 +13168,6 @@ F: Documentation/userspace-api/media/drivers/max2175.rst F: drivers/media/i2c/max2175* F: include/uapi/linux/max2175.h -MAX31827 TEMPERATURE SWITCH DRIVER -M: Daniel Matyas -L: linux-hwmon@vger.kernel.org -S: Supported -W: https://ez.analog.com/linux-software-drivers -F: Documentation/devicetree/bindings/hwmon/adi,max31827.yaml -F: Documentation/hwmon/max31827.rst -F: drivers/hwmon/max31827.c - MAX31335 RTC DRIVER M: Antoniu Miclaus L: linux-rtc@vger.kernel.org @@ -13215,15 +13182,6 @@ S: Orphan F: Documentation/hwmon/max6650.rst F: drivers/hwmon/max6650.c -MAX6697 HARDWARE MONITOR DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/devicetree/bindings/hwmon/max6697.txt -F: Documentation/hwmon/max6697.rst -F: drivers/hwmon/max6697.c -F: include/linux/platform_data/max6697.h - MAX9286 QUAD GMSL DESERIALIZER DRIVER M: Jacopo Mondi M: Kieran Bingham @@ -14629,6 +14587,14 @@ F: Documentation/driver-api/surface_aggregator/clients/dtx.rst F: drivers/platform/surface/surface_dtx.c F: include/uapi/linux/surface_aggregator/dtx.h +MICROSOFT SURFACE SENSOR FAN DRIVER +M: Maximilian Luz +M: Ivor Wanders +L: linux-hwmon@vger.kernel.org +S: Maintained +F: Documentation/hwmon/surface_fan.rst +F: drivers/hwmon/surface_fan.c + MICROSOFT SURFACE GPE LID SUPPORT DRIVER M: Maximilian Luz L: platform-driver-x86@vger.kernel.org @@ -15133,15 +15099,6 @@ M: Samuel Mendoza-Jonas S: Maintained F: net/ncsi/ -NCT6775 HARDWARE MONITOR DRIVER - CORE & PLATFORM DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/hwmon/nct6775.rst -F: drivers/hwmon/nct6775-core.c -F: drivers/hwmon/nct6775-platform.c -F: drivers/hwmon/nct6775.h - NCT6775 HARDWARE MONITOR DRIVER - I2C DRIVER M: Zev Weiss L: linux-hwmon@vger.kernel.org @@ -15897,6 +15854,14 @@ S: Maintained F: Documentation/hwmon/nzxt-kraken2.rst F: drivers/hwmon/nzxt-kraken2.c +NZXT-KRAKEN3 HARDWARE MONITORING DRIVER +M: Jonas Malaco +M: Aleksa Savic +L: linux-hwmon@vger.kernel.org +S: Maintained +F: Documentation/hwmon/nzxt-kraken3.rst +F: drivers/hwmon/nzxt-kraken3.c + NZXT-SMART2 HARDWARE MONITORING DRIVER M: Aleksandr Mezin L: linux-hwmon@vger.kernel.org @@ -17489,35 +17454,6 @@ S: Maintained F: Documentation/hwmon/pm6764tr.rst F: drivers/hwmon/pmbus/pm6764tr.c -PMBUS HARDWARE MONITORING DRIVERS -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -W: http://hwmon.wiki.kernel.org/ -W: http://www.roeck-us.net/linux/drivers/ -T: git git://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git -F: Documentation/devicetree/bindings/hwmon/ltc2978.txt -F: Documentation/devicetree/bindings/hwmon/max31785.txt -F: Documentation/hwmon/adm1275.rst -F: Documentation/hwmon/ibm-cffps.rst -F: Documentation/hwmon/ir35221.rst -F: Documentation/hwmon/lm25066.rst -F: Documentation/hwmon/ltc2978.rst -F: Documentation/hwmon/ltc3815.rst -F: Documentation/hwmon/max16064.rst -F: Documentation/hwmon/max20751.rst -F: Documentation/hwmon/max31785.rst -F: Documentation/hwmon/max34440.rst -F: Documentation/hwmon/max8688.rst -F: Documentation/hwmon/pmbus-core.rst -F: Documentation/hwmon/pmbus.rst -F: Documentation/hwmon/tps40422.rst -F: Documentation/hwmon/ucd9000.rst -F: Documentation/hwmon/ucd9200.rst -F: Documentation/hwmon/zl6100.rst -F: drivers/hwmon/pmbus/ -F: include/linux/pmbus.h - PMC SIERRA MaxRAID DRIVER L: linux-scsi@vger.kernel.org S: Orphan @@ -17739,6 +17675,13 @@ F: fs/pstore/ F: include/linux/pstore* K: \b(pstore|ramoops) +PT5161L HARDWARE MONITOR DRIVER +M: Cosmo Chou +L: linux-hwmon@vger.kernel.org +S: Maintained +F: Documentation/hwmon/pt5161l.rst +F: drivers/hwmon/pt5161l.c + PTP HARDWARE CLOCK SUPPORT M: Richard Cochran L: netdev@vger.kernel.org @@ -22226,22 +22169,6 @@ F: drivers/mmc/host/renesas_sdhi* F: drivers/mmc/host/tmio_mmc* F: include/linux/mfd/tmio.h -TMP401 HARDWARE MONITOR DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/devicetree/bindings/hwmon/ti,tmp401.yaml -F: Documentation/hwmon/tmp401.rst -F: drivers/hwmon/tmp401.c - -TMP464 HARDWARE MONITOR DRIVER -M: Guenter Roeck -L: linux-hwmon@vger.kernel.org -S: Maintained -F: Documentation/devicetree/bindings/hwmon/ti,tmp464.yaml -F: Documentation/hwmon/tmp464.rst -F: drivers/hwmon/tmp464.c - TMP513 HARDWARE MONITOR DRIVER M: Eric Tremblay L: linux-hwmon@vger.kernel.org diff --git a/arch/arm/configs/imx_v6_v7_defconfig b/arch/arm/configs/imx_v6_v7_defconfig index 92d649a817b3..7327fce87808 100644 --- a/arch/arm/configs/imx_v6_v7_defconfig +++ b/arch/arm/configs/imx_v6_v7_defconfig @@ -229,7 +229,6 @@ CONFIG_SENSORS_IIO_HWMON=y CONFIG_SENSORS_PWM_FAN=y CONFIG_SENSORS_SY7636A=y CONFIG_THERMAL_STATISTICS=y -CONFIG_THERMAL_WRITABLE_TRIPS=y CONFIG_CPU_THERMAL=y CONFIG_IMX_THERMAL=y CONFIG_WATCHDOG=y diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 89afacce6951..78050d5d7fac 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1065,8 +1065,9 @@ config SCHED_MC config SCHED_MC_PRIO bool "CPU core priorities scheduler support" - depends on SCHED_MC && CPU_SUP_INTEL - select X86_INTEL_PSTATE + depends on SCHED_MC + select X86_INTEL_PSTATE if CPU_SUP_INTEL + select X86_AMD_PSTATE if CPU_SUP_AMD && ACPI select CPU_FREQ default y help diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c index 8d8752b44f11..ff8f25faca3d 100644 --- a/arch/x86/kernel/acpi/cppc.c +++ b/arch/x86/kernel/acpi/cppc.c @@ -20,7 +20,7 @@ bool cpc_supported_by_cpu(void) (boot_cpu_data.x86_model >= 0x20 && boot_cpu_data.x86_model <= 0x2f))) return true; else if (boot_cpu_data.x86 == 0x17 && - boot_cpu_data.x86_model >= 0x70 && boot_cpu_data.x86_model <= 0x7f) + boot_cpu_data.x86_model >= 0x30 && boot_cpu_data.x86_model <= 0x7f) return true; return boot_cpu_has(X86_FEATURE_CPPC); } diff --git a/arch/x86/kernel/acpi/cstate.c b/arch/x86/kernel/acpi/cstate.c index 401808b47af3..f3ffd0a3a012 100644 --- a/arch/x86/kernel/acpi/cstate.c +++ b/arch/x86/kernel/acpi/cstate.c @@ -131,8 +131,8 @@ static long acpi_processor_ffh_cstate_probe_cpu(void *_cx) cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx); /* Check whether this particular cx_type (in CST) is supported or not */ - cstate_type = ((cx->address >> MWAIT_SUBSTATE_SIZE) & - MWAIT_CSTATE_MASK) + 1; + cstate_type = (((cx->address >> MWAIT_SUBSTATE_SIZE) & + MWAIT_CSTATE_MASK) + 1) & MWAIT_CSTATE_MASK; edx_part = edx >> (cstate_type * MWAIT_SUBSTATE_SIZE); num_cstate_subtype = edx_part & MWAIT_SUBSTATE_MASK; diff --git a/drivers/accel/ivpu/ivpu_pm.c b/drivers/accel/ivpu/ivpu_pm.c index 5f73854234ba..814882ae5059 100644 --- a/drivers/accel/ivpu/ivpu_pm.c +++ b/drivers/accel/ivpu/ivpu_pm.c @@ -309,7 +309,7 @@ int ivpu_rpm_get_if_active(struct ivpu_device *vdev) { int ret; - ret = pm_runtime_get_if_active(vdev->drm.dev, false); + ret = pm_runtime_get_if_in_use(vdev->drm.dev); drm_WARN_ON(&vdev->drm, ret < 0); return ret; diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index 3c3f8037ebed..c645bb453f3b 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -449,20 +449,6 @@ config ACPI_HED which is used to report some hardware errors notified via SCI, mainly the corrected errors. -config ACPI_CUSTOM_METHOD - tristate "Allow ACPI methods to be inserted/replaced at run time" - depends on DEBUG_FS - help - This debug facility allows ACPI AML methods to be inserted and/or - replaced without rebooting the system. For details refer to: - Documentation/firmware-guide/acpi/method-customizing.rst. - - NOTE: This option is security sensitive, because it allows arbitrary - kernel memory to be written to by root (uid=0) users, allowing them - to bypass certain security measures (e.g. if root is not allowed to - load additional kernel modules after boot, this feature may be used - to override that restriction). - config ACPI_BGRT bool "Boottime Graphics Resource Table support" depends on EFI && (X86 || ARM64) diff --git a/drivers/acpi/Makefile b/drivers/acpi/Makefile index 12ef8180d272..8cc8c0d9c873 100644 --- a/drivers/acpi/Makefile +++ b/drivers/acpi/Makefile @@ -101,7 +101,6 @@ obj-$(CONFIG_ACPI_SBS) += sbshc.o obj-$(CONFIG_ACPI_SBS) += sbs.o obj-$(CONFIG_ACPI_HED) += hed.o obj-$(CONFIG_ACPI_EC_DEBUGFS) += ec_sys.o -obj-$(CONFIG_ACPI_CUSTOM_METHOD)+= custom_method.o obj-$(CONFIG_ACPI_BGRT) += bgrt.o obj-$(CONFIG_ACPI_CPPC_LIB) += cppc_acpi.o obj-$(CONFIG_ACPI_SPCR_TABLE) += spcr.o diff --git a/drivers/acpi/acpi_processor.c b/drivers/acpi/acpi_processor.c index 4fe2ef54088c..7a0dd35d62c9 100644 --- a/drivers/acpi/acpi_processor.c +++ b/drivers/acpi/acpi_processor.c @@ -161,7 +161,7 @@ static void cpufreq_add_device(const char *name) pdev = platform_device_register_simple(name, PLATFORM_DEVID_NONE, NULL, 0); if (IS_ERR(pdev)) - pr_info("%s device creation failed: %ld\n", name, PTR_ERR(pdev)); + pr_info("%s device creation failed: %pe\n", name, pdev); } #ifdef CONFIG_X86 @@ -381,6 +381,9 @@ static int acpi_processor_add(struct acpi_device *device, struct device *dev; int result = 0; + if (!acpi_device_is_enabled(device)) + return -ENODEV; + pr = kzalloc(sizeof(struct acpi_processor), GFP_KERNEL); if (!pr) return -ENOMEM; diff --git a/drivers/acpi/acpi_tad.c b/drivers/acpi/acpi_tad.c index 33c3b16af556..1d670dbe4d1d 100644 --- a/drivers/acpi/acpi_tad.c +++ b/drivers/acpi/acpi_tad.c @@ -554,7 +554,7 @@ static int acpi_tad_disable_timer(struct device *dev, u32 timer_id) return acpi_tad_wake_set(dev, "_STV", timer_id, ACPI_TAD_WAKE_DISABLED); } -static int acpi_tad_remove(struct platform_device *pdev) +static void acpi_tad_remove(struct platform_device *pdev) { struct device *dev = &pdev->dev; acpi_handle handle = ACPI_HANDLE(dev); @@ -579,7 +579,6 @@ static int acpi_tad_remove(struct platform_device *pdev) pm_runtime_put_sync(dev); pm_runtime_disable(dev); acpi_remove_cmos_rtc_space_handler(handle); - return 0; } static int acpi_tad_probe(struct platform_device *pdev) @@ -684,7 +683,7 @@ static struct platform_driver acpi_tad_driver = { .acpi_match_table = acpi_tad_ids, }, .probe = acpi_tad_probe, - .remove = acpi_tad_remove, + .remove_new = acpi_tad_remove, }; MODULE_DEVICE_TABLE(acpi, acpi_tad_ids); diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c index 4afdda9db019..1fda30388297 100644 --- a/drivers/acpi/acpi_video.c +++ b/drivers/acpi/acpi_video.c @@ -612,7 +612,7 @@ acpi_video_device_lcd_get_level_current(struct acpi_video_device *device, static int acpi_video_device_EDID(struct acpi_video_device *device, - union acpi_object **edid, ssize_t length) + union acpi_object **edid, int length) { int status; struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; @@ -625,13 +625,11 @@ acpi_video_device_EDID(struct acpi_video_device *device, if (!device) return -ENODEV; - if (length == 128) - arg0.integer.value = 1; - else if (length == 256) - arg0.integer.value = 2; - else + if (!length || (length % 128)) return -EINVAL; + arg0.integer.value = length / 128; + status = acpi_evaluate_object(device->dev->handle, "_DDC", &args, &buffer); if (ACPI_FAILURE(status)) return -ENODEV; @@ -641,7 +639,8 @@ acpi_video_device_EDID(struct acpi_video_device *device, if (obj && obj->type == ACPI_TYPE_BUFFER) *edid = obj; else { - acpi_handle_info(device->dev->handle, "Invalid _DDC data\n"); + acpi_handle_debug(device->dev->handle, + "Invalid _DDC data for length %d\n", length); status = -EFAULT; kfree(obj); } @@ -1447,7 +1446,6 @@ int acpi_video_get_edid(struct acpi_device *device, int type, int device_id, for (i = 0; i < video->attached_count; i++) { video_device = video->attached_array[i].bind_info; - length = 256; if (!video_device) continue; @@ -1478,18 +1476,14 @@ int acpi_video_get_edid(struct acpi_device *device, int type, int device_id, continue; } - status = acpi_video_device_EDID(video_device, &buffer, length); - - if (ACPI_FAILURE(status) || !buffer || - buffer->type != ACPI_TYPE_BUFFER) { - length = 128; + for (length = 512; length > 0; length -= 128) { status = acpi_video_device_EDID(video_device, &buffer, length); - if (ACPI_FAILURE(status) || !buffer || - buffer->type != ACPI_TYPE_BUFFER) { - continue; - } + if (ACPI_SUCCESS(status)) + break; } + if (!length) + continue; *edid = buffer->buffer.pointer; return length; diff --git a/drivers/acpi/acpi_watchdog.c b/drivers/acpi/acpi_watchdog.c index 8e9e001da38f..14b24157799c 100644 --- a/drivers/acpi/acpi_watchdog.c +++ b/drivers/acpi/acpi_watchdog.c @@ -179,7 +179,7 @@ void __init acpi_watchdog_init(void) pdev = platform_device_register_simple("wdat_wdt", PLATFORM_DEVID_NONE, resources, nresources); if (IS_ERR(pdev)) - pr_err("Device creation failed: %ld\n", PTR_ERR(pdev)); + pr_err("Device creation failed: %pe\n", pdev); kfree(resources); diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index ab2a82cb1b0b..512067cac170 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -1455,7 +1455,7 @@ err: return rc; } -static int ghes_remove(struct platform_device *ghes_dev) +static void ghes_remove(struct platform_device *ghes_dev) { int rc; struct ghes *ghes; @@ -1492,8 +1492,15 @@ static int ghes_remove(struct platform_device *ghes_dev) break; case ACPI_HEST_NOTIFY_SOFTWARE_DELEGATED: rc = apei_sdei_unregister_ghes(ghes); - if (rc) - return rc; + if (rc) { + /* + * Returning early results in a resource leak, but we're + * only here if stopping the hardware failed. + */ + dev_err(&ghes_dev->dev, "Failed to unregister ghes (%pe)\n", + ERR_PTR(rc)); + return; + } break; default: BUG(); @@ -1507,8 +1514,6 @@ static int ghes_remove(struct platform_device *ghes_dev) mutex_unlock(&ghes_devs_mutex); kfree(ghes); - - return 0; } static struct platform_driver ghes_platform_driver = { @@ -1516,7 +1521,7 @@ static struct platform_driver ghes_platform_driver = { .name = "GHES", }, .probe = ghes_probe, - .remove = ghes_remove, + .remove_new = ghes_remove, }; void __init acpi_ghes_init(void) diff --git a/drivers/acpi/apei/hest.c b/drivers/acpi/apei/hest.c index 6aef1ee5e1bd..20d757687e3d 100644 --- a/drivers/acpi/apei/hest.c +++ b/drivers/acpi/apei/hest.c @@ -37,6 +37,20 @@ EXPORT_SYMBOL_GPL(hest_disable); static struct acpi_table_hest *__read_mostly hest_tab; +/* + * Since GHES_ASSIST is not supported, skip initialization of GHES_ASSIST + * structures for MCA. + * During HEST parsing, detected MCA error sources are cached from early + * table entries so that the Flags and Source Id fields from these cached + * values are then referred to in later table entries to determine if the + * encountered GHES_ASSIST structure should be initialized. + */ +static struct { + struct acpi_hest_ia_corrected *cmc; + struct acpi_hest_ia_machine_check *mc; + struct acpi_hest_ia_deferred_check *dmc; +} mces; + static const int hest_esrc_len_tab[ACPI_HEST_TYPE_RESERVED] = { [ACPI_HEST_TYPE_IA32_CHECK] = -1, /* need further calculation */ [ACPI_HEST_TYPE_IA32_CORRECTED_CHECK] = -1, @@ -70,22 +84,54 @@ static int hest_esrc_len(struct acpi_hest_header *hest_hdr) cmc = (struct acpi_hest_ia_corrected *)hest_hdr; len = sizeof(*cmc) + cmc->num_hardware_banks * sizeof(struct acpi_hest_ia_error_bank); + mces.cmc = cmc; } else if (hest_type == ACPI_HEST_TYPE_IA32_CHECK) { struct acpi_hest_ia_machine_check *mc; mc = (struct acpi_hest_ia_machine_check *)hest_hdr; len = sizeof(*mc) + mc->num_hardware_banks * sizeof(struct acpi_hest_ia_error_bank); + mces.mc = mc; } else if (hest_type == ACPI_HEST_TYPE_IA32_DEFERRED_CHECK) { struct acpi_hest_ia_deferred_check *mc; mc = (struct acpi_hest_ia_deferred_check *)hest_hdr; len = sizeof(*mc) + mc->num_hardware_banks * sizeof(struct acpi_hest_ia_error_bank); + mces.dmc = mc; } BUG_ON(len == -1); return len; }; +/* + * GHES and GHESv2 structures share the same format, starting from + * Source Id and ending in Error Status Block Length (inclusive). + */ +static bool is_ghes_assist_struct(struct acpi_hest_header *hest_hdr) +{ + struct acpi_hest_generic *ghes; + u16 related_source_id; + + if (hest_hdr->type != ACPI_HEST_TYPE_GENERIC_ERROR && + hest_hdr->type != ACPI_HEST_TYPE_GENERIC_ERROR_V2) + return false; + + ghes = (struct acpi_hest_generic *)hest_hdr; + related_source_id = ghes->related_source_id; + + if (mces.cmc && mces.cmc->flags & ACPI_HEST_GHES_ASSIST && + related_source_id == mces.cmc->header.source_id) + return true; + if (mces.mc && mces.mc->flags & ACPI_HEST_GHES_ASSIST && + related_source_id == mces.mc->header.source_id) + return true; + if (mces.dmc && mces.dmc->flags & ACPI_HEST_GHES_ASSIST && + related_source_id == mces.dmc->header.source_id) + return true; + + return false; +} + typedef int (*apei_hest_func_t)(struct acpi_hest_header *hest_hdr, void *data); static int apei_hest_parse(apei_hest_func_t func, void *data) @@ -114,6 +160,11 @@ static int apei_hest_parse(apei_hest_func_t func, void *data) return -EINVAL; } + if (is_ghes_assist_struct(hest_hdr)) { + hest_hdr = (void *)hest_hdr + len; + continue; + } + rc = func(hest_hdr, data); if (rc) return rc; diff --git a/drivers/acpi/arm64/agdi.c b/drivers/acpi/arm64/agdi.c index 8b3c7d42b41b..f5f21dd0d277 100644 --- a/drivers/acpi/arm64/agdi.c +++ b/drivers/acpi/arm64/agdi.c @@ -58,7 +58,7 @@ static int agdi_probe(struct platform_device *pdev) return agdi_sdei_probe(pdev, adata); } -static int agdi_remove(struct platform_device *pdev) +static void agdi_remove(struct platform_device *pdev) { struct agdi_data *adata = dev_get_platdata(&pdev->dev); int err, i; @@ -67,7 +67,7 @@ static int agdi_remove(struct platform_device *pdev) if (err) { dev_err(&pdev->dev, "Failed to disable sdei-event #%d (%pe)\n", adata->sdei_event, ERR_PTR(err)); - return 0; + return; } for (i = 0; i < 3; i++) { @@ -81,8 +81,6 @@ static int agdi_remove(struct platform_device *pdev) if (err) dev_err(&pdev->dev, "Failed to unregister sdei-event #%d (%pe)\n", adata->sdei_event, ERR_PTR(err)); - - return 0; } static struct platform_driver agdi_driver = { @@ -90,7 +88,7 @@ static struct platform_driver agdi_driver = { .name = "agdi", }, .probe = agdi_probe, - .remove = agdi_remove, + .remove_new = agdi_remove, }; void __init acpi_agdi_init(void) diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c index 569bd15f211b..d9fa730416f1 100644 --- a/drivers/acpi/bus.c +++ b/drivers/acpi/bus.c @@ -1097,7 +1097,7 @@ static void acpi_device_remove(struct device *dev) put_device(dev); } -struct bus_type acpi_bus_type = { +const struct bus_type acpi_bus_type = { .name = "acpi", .match = acpi_bus_match, .probe = acpi_device_probe, diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index d155a86a8614..4bfbe55553f4 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -166,6 +166,13 @@ show_cppc_data(cppc_get_perf_caps, cppc_perf_caps, nominal_freq); show_cppc_data(cppc_get_perf_ctrs, cppc_perf_fb_ctrs, reference_perf); show_cppc_data(cppc_get_perf_ctrs, cppc_perf_fb_ctrs, wraparound_time); +/* Check for valid access_width, otherwise, fallback to using bit_width */ +#define GET_BIT_WIDTH(reg) ((reg)->access_width ? (8 << ((reg)->access_width - 1)) : (reg)->bit_width) + +/* Shift and apply the mask for CPC reads/writes */ +#define MASK_VAL(reg, val) ((val) >> ((reg)->bit_offset & \ + GENMASK(((reg)->bit_width), 0))) + static ssize_t show_feedback_ctrs(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -780,6 +787,7 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) } else if (gas_t->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) { if (gas_t->address) { void __iomem *addr; + size_t access_width; if (!osc_cpc_flexible_adr_space_confirmed) { pr_debug("Flexible address space capability not supported\n"); @@ -787,7 +795,8 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) goto out_free; } - addr = ioremap(gas_t->address, gas_t->bit_width/8); + access_width = GET_BIT_WIDTH(gas_t) / 8; + addr = ioremap(gas_t->address, access_width); if (!addr) goto out_free; cpc_ptr->cpc_regs[i-2].sys_mem_vaddr = addr; @@ -983,6 +992,7 @@ int __weak cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) static int cpc_read(int cpu, struct cpc_register_resource *reg_res, u64 *val) { void __iomem *vaddr = NULL; + int size; int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu); struct cpc_reg *reg = ®_res->cpc_entry.reg; @@ -994,7 +1004,7 @@ static int cpc_read(int cpu, struct cpc_register_resource *reg_res, u64 *val) *val = 0; if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_IO) { - u32 width = 8 << (reg->access_width - 1); + u32 width = GET_BIT_WIDTH(reg); u32 val_u32; acpi_status status; @@ -1018,7 +1028,9 @@ static int cpc_read(int cpu, struct cpc_register_resource *reg_res, u64 *val) return acpi_os_read_memory((acpi_physical_address)reg->address, val, reg->bit_width); - switch (reg->bit_width) { + size = GET_BIT_WIDTH(reg); + + switch (size) { case 8: *val = readb_relaxed(vaddr); break; @@ -1037,18 +1049,22 @@ static int cpc_read(int cpu, struct cpc_register_resource *reg_res, u64 *val) return -EFAULT; } + if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) + *val = MASK_VAL(reg, *val); + return 0; } static int cpc_write(int cpu, struct cpc_register_resource *reg_res, u64 val) { int ret_val = 0; + int size; void __iomem *vaddr = NULL; int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu); struct cpc_reg *reg = ®_res->cpc_entry.reg; if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_IO) { - u32 width = 8 << (reg->access_width - 1); + u32 width = GET_BIT_WIDTH(reg); acpi_status status; status = acpi_os_write_port((acpi_io_address)reg->address, @@ -1070,7 +1086,12 @@ static int cpc_write(int cpu, struct cpc_register_resource *reg_res, u64 val) return acpi_os_write_memory((acpi_physical_address)reg->address, val, reg->bit_width); - switch (reg->bit_width) { + size = GET_BIT_WIDTH(reg); + + if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) + val = MASK_VAL(reg, val); + + switch (size) { case 8: writeb_relaxed(val, vaddr); break; @@ -1157,6 +1178,19 @@ int cppc_get_nominal_perf(int cpunum, u64 *nominal_perf) return cppc_get_perf(cpunum, NOMINAL_PERF, nominal_perf); } +/** + * cppc_get_highest_perf - Get the highest performance register value. + * @cpunum: CPU from which to get highest performance. + * @highest_perf: Return address. + * + * Return: 0 for success, -EIO otherwise. + */ +int cppc_get_highest_perf(int cpunum, u64 *highest_perf) +{ + return cppc_get_perf(cpunum, HIGHEST_PERF, highest_perf); +} +EXPORT_SYMBOL_GPL(cppc_get_highest_perf); + /** * cppc_get_epp_perf - Get the epp register value. * @cpunum: CPU from which to get epp preference value. diff --git a/drivers/acpi/custom_method.c b/drivers/acpi/custom_method.c deleted file mode 100644 index d39a9b474727..000000000000 --- a/drivers/acpi/custom_method.c +++ /dev/null @@ -1,103 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * custom_method.c - debugfs interface for customizing ACPI control method - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "internal.h" - -MODULE_LICENSE("GPL"); - -static struct dentry *cm_dentry; - -/* /sys/kernel/debug/acpi/custom_method */ - -static ssize_t cm_write(struct file *file, const char __user *user_buf, - size_t count, loff_t *ppos) -{ - static char *buf; - static u32 max_size; - static u32 uncopied_bytes; - - struct acpi_table_header table; - acpi_status status; - int ret; - - ret = security_locked_down(LOCKDOWN_ACPI_TABLES); - if (ret) - return ret; - - if (!(*ppos)) { - /* parse the table header to get the table length */ - if (count <= sizeof(struct acpi_table_header)) - return -EINVAL; - if (copy_from_user(&table, user_buf, - sizeof(struct acpi_table_header))) - return -EFAULT; - uncopied_bytes = max_size = table.length; - /* make sure the buf is not allocated */ - kfree(buf); - buf = kzalloc(max_size, GFP_KERNEL); - if (!buf) - return -ENOMEM; - } - - if (buf == NULL) - return -EINVAL; - - if ((*ppos > max_size) || - (*ppos + count > max_size) || - (*ppos + count < count) || - (count > uncopied_bytes)) { - kfree(buf); - buf = NULL; - return -EINVAL; - } - - if (copy_from_user(buf + (*ppos), user_buf, count)) { - kfree(buf); - buf = NULL; - return -EFAULT; - } - - uncopied_bytes -= count; - *ppos += count; - - if (!uncopied_bytes) { - status = acpi_install_method(buf); - kfree(buf); - buf = NULL; - if (ACPI_FAILURE(status)) - return -EINVAL; - add_taint(TAINT_OVERRIDDEN_ACPI_TABLE, LOCKDEP_NOW_UNRELIABLE); - } - - return count; -} - -static const struct file_operations cm_fops = { - .write = cm_write, - .llseek = default_llseek, -}; - -static int __init acpi_custom_method_init(void) -{ - cm_dentry = debugfs_create_file("custom_method", S_IWUSR, - acpi_debugfs_dir, NULL, &cm_fops); - return 0; -} - -static void __exit acpi_custom_method_exit(void) -{ - debugfs_remove(cm_dentry); -} - -module_init(acpi_custom_method_init); -module_exit(acpi_custom_method_exit); diff --git a/drivers/acpi/dptf/dptf_pch_fivr.c b/drivers/acpi/dptf/dptf_pch_fivr.c index 4919e7abe93f..654aaa53c67f 100644 --- a/drivers/acpi/dptf/dptf_pch_fivr.c +++ b/drivers/acpi/dptf/dptf_pch_fivr.c @@ -141,11 +141,9 @@ static int pch_fivr_add(struct platform_device *pdev) return 0; } -static int pch_fivr_remove(struct platform_device *pdev) +static void pch_fivr_remove(struct platform_device *pdev) { sysfs_remove_group(&pdev->dev.kobj, &pch_fivr_attribute_group); - - return 0; } static const struct acpi_device_id pch_fivr_device_ids[] = { @@ -159,7 +157,7 @@ MODULE_DEVICE_TABLE(acpi, pch_fivr_device_ids); static struct platform_driver pch_fivr_driver = { .probe = pch_fivr_add, - .remove = pch_fivr_remove, + .remove_new = pch_fivr_remove, .driver = { .name = "dptf_pch_fivr", .acpi_match_table = pch_fivr_device_ids, diff --git a/drivers/acpi/dptf/dptf_power.c b/drivers/acpi/dptf/dptf_power.c index 86561eda939f..b8187babbbbb 100644 --- a/drivers/acpi/dptf/dptf_power.c +++ b/drivers/acpi/dptf/dptf_power.c @@ -209,7 +209,7 @@ static int dptf_power_add(struct platform_device *pdev) return 0; } -static int dptf_power_remove(struct platform_device *pdev) +static void dptf_power_remove(struct platform_device *pdev) { struct acpi_device *acpi_dev = platform_get_drvdata(pdev); @@ -221,8 +221,6 @@ static int dptf_power_remove(struct platform_device *pdev) sysfs_remove_group(&pdev->dev.kobj, &dptf_battery_attribute_group); else sysfs_remove_group(&pdev->dev.kobj, &dptf_power_attribute_group); - - return 0; } static const struct acpi_device_id int3407_device_ids[] = { @@ -242,7 +240,7 @@ MODULE_DEVICE_TABLE(acpi, int3407_device_ids); static struct platform_driver dptf_power_driver = { .probe = dptf_power_add, - .remove = dptf_power_remove, + .remove_new = dptf_power_remove, .driver = { .name = "dptf_power", .acpi_match_table = int3407_device_ids, diff --git a/drivers/acpi/evged.c b/drivers/acpi/evged.c index fe6b6792c8bb..11778c93254b 100644 --- a/drivers/acpi/evged.c +++ b/drivers/acpi/evged.c @@ -173,10 +173,9 @@ static void ged_shutdown(struct platform_device *pdev) } } -static int ged_remove(struct platform_device *pdev) +static void ged_remove(struct platform_device *pdev) { ged_shutdown(pdev); - return 0; } static const struct acpi_device_id ged_acpi_ids[] = { @@ -186,7 +185,7 @@ static const struct acpi_device_id ged_acpi_ids[] = { static struct platform_driver ged_driver = { .probe = ged_probe, - .remove = ged_remove, + .remove_new = ged_remove, .shutdown = ged_shutdown, .driver = { .name = MODULE_NAME, diff --git a/drivers/acpi/fan_core.c b/drivers/acpi/fan_core.c index 9dccbae9e8ea..ff72e4ef8738 100644 --- a/drivers/acpi/fan_core.c +++ b/drivers/acpi/fan_core.c @@ -387,7 +387,7 @@ err_end: return result; } -static int acpi_fan_remove(struct platform_device *pdev) +static void acpi_fan_remove(struct platform_device *pdev) { struct acpi_fan *fan = platform_get_drvdata(pdev); @@ -399,8 +399,6 @@ static int acpi_fan_remove(struct platform_device *pdev) sysfs_remove_link(&pdev->dev.kobj, "thermal_cooling"); sysfs_remove_link(&fan->cdev->device.kobj, "device"); thermal_cooling_device_unregister(fan->cdev); - - return 0; } #ifdef CONFIG_PM_SLEEP @@ -446,7 +444,7 @@ static const struct dev_pm_ops acpi_fan_pm = { static struct platform_driver acpi_fan_driver = { .probe = acpi_fan_probe, - .remove = acpi_fan_remove, + .remove_new = acpi_fan_remove, .driver = { .name = "acpi-fan", .acpi_match_table = fan_device_ids, diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h index 6588525c45ef..ca72a0dc5715 100644 --- a/drivers/acpi/internal.h +++ b/drivers/acpi/internal.h @@ -121,6 +121,7 @@ int acpi_device_setup_files(struct acpi_device *dev); void acpi_device_remove_files(struct acpi_device *dev); void acpi_device_add_finalize(struct acpi_device *device); void acpi_free_pnp_ids(struct acpi_device_pnp *pnp); +bool acpi_device_is_enabled(const struct acpi_device *adev); bool acpi_device_is_present(const struct acpi_device *adev); bool acpi_device_is_battery(struct acpi_device *adev); bool acpi_device_is_first_physical_node(struct acpi_device *adev, @@ -301,5 +302,6 @@ void acpi_mipi_check_crs_csi2(acpi_handle handle); void acpi_mipi_scan_crs_csi2(void); void acpi_mipi_init_crs_csi2_swnodes(void); void acpi_mipi_crs_csi2_cleanup(void); +bool acpi_graph_ignore_port(acpi_handle handle); #endif /* _ACPI_INTERNAL_H_ */ diff --git a/drivers/acpi/mipi-disco-img.c b/drivers/acpi/mipi-disco-img.c index 7286cf4579bc..d05413a0672a 100644 --- a/drivers/acpi/mipi-disco-img.c +++ b/drivers/acpi/mipi-disco-img.c @@ -19,6 +19,7 @@ */ #include +#include #include #include #include @@ -723,3 +724,73 @@ void acpi_mipi_crs_csi2_cleanup(void) list_for_each_entry_safe(csi2, csi2_tmp, &acpi_mipi_crs_csi2_list, entry) acpi_mipi_del_crs_csi2(csi2); } + +static const struct dmi_system_id dmi_ignore_port_nodes[] = { + { + .matches = { + DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "XPS 9315"), + }, + }, + { } +}; + +static const char *strnext(const char *s1, const char *s2) +{ + s1 = strstr(s1, s2); + + if (!s1) + return NULL; + + return s1 + strlen(s2); +} + +/** + * acpi_graph_ignore_port - Tell whether a port node should be ignored + * @handle: The ACPI handle of the node (which may be a port node) + * + * Return: true if a port node should be ignored and the data to that should + * come from other sources instead (Windows ACPI definitions and + * ipu-bridge). This is currently used to ignore bad port nodes related to IPU6 + * ("IPU?") and camera sensor devices ("LNK?") in certain Dell systems with + * Intel VSC. + */ +bool acpi_graph_ignore_port(acpi_handle handle) +{ + const char *path = NULL, *orig_path; + static bool dmi_tested, ignore_port; + + if (!dmi_tested) { + ignore_port = dmi_first_match(dmi_ignore_port_nodes); + dmi_tested = true; + } + + if (!ignore_port) + return false; + + /* Check if the device is either "IPU" or "LNK" (sensor). */ + orig_path = acpi_handle_path(handle); + if (!orig_path) + return false; + path = strnext(orig_path, "IPU"); + if (!path) + path = strnext(orig_path, "LNK"); + if (!path) + goto out_free; + + if (!(isdigit(path[0]) && path[1] == '.')) + goto out_free; + + /* Check if the node has a "PRT" prefix. */ + path = strnext(path, "PRT"); + if (path && isdigit(path[0]) && !path[1]) { + acpi_handle_debug(handle, "ignoring data node\n"); + + kfree(orig_path); + return true; + } + +out_free: + kfree(orig_path); + return false; +} diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c index 802f8a56d1fa..d4595d1985b1 100644 --- a/drivers/acpi/nfit/core.c +++ b/drivers/acpi/nfit/core.c @@ -1737,9 +1737,8 @@ __weak void nfit_intel_shutdown_status(struct nfit_mem *nfit_mem) if ((nfit_mem->dsm_mask & (1 << func)) == 0) return; - out_obj = acpi_evaluate_dsm(handle, guid, revid, func, &in_obj); - if (!out_obj || out_obj->type != ACPI_TYPE_BUFFER - || out_obj->buffer.length < sizeof(smart)) { + out_obj = acpi_evaluate_dsm_typed(handle, guid, revid, func, &in_obj, ACPI_TYPE_BUFFER); + if (!out_obj || out_obj->buffer.length < sizeof(smart)) { dev_dbg(dev->parent, "%s: failed to retrieve initial health\n", dev_name(dev)); ACPI_FREE(out_obj); diff --git a/drivers/acpi/pci_slot.c b/drivers/acpi/pci_slot.c index d6cb2c27a23b..741bcc9d6d6a 100644 --- a/drivers/acpi/pci_slot.c +++ b/drivers/acpi/pci_slot.c @@ -111,7 +111,7 @@ register_slot(acpi_handle handle, u32 lvl, void *context, void **rv) snprintf(name, sizeof(name), "%llu", sun); pci_slot = pci_create_slot(pci_bus, device, name, NULL); if (IS_ERR(pci_slot)) { - pr_err("pci_create_slot returned %ld\n", PTR_ERR(pci_slot)); + pr_err("pci_create_slot returned %pe\n", pci_slot); kfree(slot); return AE_OK; } diff --git a/drivers/acpi/pfr_telemetry.c b/drivers/acpi/pfr_telemetry.c index 843f678ade0c..998264a7333d 100644 --- a/drivers/acpi/pfr_telemetry.c +++ b/drivers/acpi/pfr_telemetry.c @@ -347,13 +347,11 @@ static const struct file_operations acpi_pfrt_log_fops = { .llseek = noop_llseek, }; -static int acpi_pfrt_log_remove(struct platform_device *pdev) +static void acpi_pfrt_log_remove(struct platform_device *pdev) { struct pfrt_log_device *pfrt_log_dev = platform_get_drvdata(pdev); misc_deregister(&pfrt_log_dev->miscdev); - - return 0; } static void pfrt_log_put_idx(void *data) @@ -427,7 +425,7 @@ static struct platform_driver acpi_pfrt_log_driver = { .acpi_match_table = acpi_pfrt_log_ids, }, .probe = acpi_pfrt_log_probe, - .remove = acpi_pfrt_log_remove, + .remove_new = acpi_pfrt_log_remove, }; module_platform_driver(acpi_pfrt_log_driver); diff --git a/drivers/acpi/pfr_update.c b/drivers/acpi/pfr_update.c index 98267f163e2b..8b2910995fc1 100644 --- a/drivers/acpi/pfr_update.c +++ b/drivers/acpi/pfr_update.c @@ -489,13 +489,11 @@ static const struct file_operations acpi_pfru_fops = { .llseek = noop_llseek, }; -static int acpi_pfru_remove(struct platform_device *pdev) +static void acpi_pfru_remove(struct platform_device *pdev) { struct pfru_device *pfru_dev = platform_get_drvdata(pdev); misc_deregister(&pfru_dev->miscdev); - - return 0; } static void pfru_put_idx(void *data) @@ -567,7 +565,7 @@ static struct platform_driver acpi_pfru_driver = { .acpi_match_table = acpi_pfru_ids, }, .probe = acpi_pfru_probe, - .remove = acpi_pfru_remove, + .remove_new = acpi_pfru_remove, }; module_platform_driver(acpi_pfru_driver); diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c index 4bd16b3f0781..67db60eda370 100644 --- a/drivers/acpi/processor_driver.c +++ b/drivers/acpi/processor_driver.c @@ -27,6 +27,7 @@ #define ACPI_PROCESSOR_NOTIFY_PERFORMANCE 0x80 #define ACPI_PROCESSOR_NOTIFY_POWER 0x81 #define ACPI_PROCESSOR_NOTIFY_THROTTLING 0x82 +#define ACPI_PROCESSOR_NOTIFY_HIGEST_PERF_CHANGED 0x85 MODULE_AUTHOR("Paul Diefenbaugh"); MODULE_DESCRIPTION("ACPI Processor Driver"); @@ -83,6 +84,11 @@ static void acpi_processor_notify(acpi_handle handle, u32 event, void *data) acpi_bus_generate_netlink_event(device->pnp.device_class, dev_name(&device->dev), event, 0); break; + case ACPI_PROCESSOR_NOTIFY_HIGEST_PERF_CHANGED: + cpufreq_update_limits(pr->id); + acpi_bus_generate_netlink_event(device->pnp.device_class, + dev_name(&device->dev), event, 0); + break; default: acpi_handle_debug(handle, "Unsupported event [0x%x]\n", event); break; diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index 55437f5e0c3a..bd6a7857ce05 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -1430,6 +1430,8 @@ int acpi_processor_power_exit(struct acpi_processor *pr) acpi_processor_registered--; if (acpi_processor_registered == 0) cpuidle_unregister_driver(&acpi_idle_driver); + + kfree(dev); } pr->flags.power_setup_done = 0; diff --git a/drivers/acpi/property.c b/drivers/acpi/property.c index a6ead5204046..2b73580c9f36 100644 --- a/drivers/acpi/property.c +++ b/drivers/acpi/property.c @@ -80,6 +80,9 @@ static bool acpi_nondev_subnode_extract(union acpi_object *desc, struct acpi_data_node *dn; bool result; + if (acpi_graph_ignore_port(handle)) + return false; + dn = kzalloc(sizeof(*dn), GFP_KERNEL); if (!dn) return false; diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c index dacad1d846c0..59423fe9d0f2 100644 --- a/drivers/acpi/resource.c +++ b/drivers/acpi/resource.c @@ -468,6 +468,13 @@ static const struct dmi_system_id irq1_level_low_skip_override[] = { DMI_MATCH(DMI_BOARD_NAME, "B1502CGA"), }, }, + { + /* Asus ExpertBook B1502CVA */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_BOARD_NAME, "B1502CVA"), + }, + }, { /* Asus ExpertBook B2402CBA */ .matches = { @@ -489,6 +496,13 @@ static const struct dmi_system_id irq1_level_low_skip_override[] = { DMI_MATCH(DMI_BOARD_NAME, "B2502CBA"), }, }, + { + /* Asus ExpertBook B2502FBA */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_BOARD_NAME, "B2502FBA"), + }, + }, { /* Asus Vivobook E1504GA */ .matches = { @@ -588,6 +602,34 @@ static const struct dmi_system_id irq1_edge_low_force_override[] = { DMI_MATCH(DMI_BOARD_NAME, "GM5RGEE0016COM"), }, }, + { + /* Lunnen Ground 15 / AMD Ryzen 5 5500U */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Lunnen"), + DMI_MATCH(DMI_BOARD_NAME, "LLL5DAW"), + }, + }, + { + /* Lunnen Ground 16 / AMD Ryzen 7 5800U */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Lunnen"), + DMI_MATCH(DMI_BOARD_NAME, "LL6FA"), + }, + }, + { + /* MAIBENBEN X577 */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "MAIBENBEN"), + DMI_MATCH(DMI_BOARD_NAME, "X577"), + }, + }, + { + /* Maibenben X565 */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "MAIBENBEN"), + DMI_MATCH(DMI_BOARD_NAME, "X565"), + }, + }, { } }; diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index e6ed1ba91e5c..3b722e4c0f06 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -244,6 +244,53 @@ static int acpi_scan_try_to_offline(struct acpi_device *device) return 0; } +static int acpi_scan_check_and_detach(struct acpi_device *adev, void *check) +{ + struct acpi_scan_handler *handler = adev->handler; + + acpi_dev_for_each_child_reverse(adev, acpi_scan_check_and_detach, check); + + if (check) { + acpi_bus_get_status(adev); + /* + * Skip devices that are still there and take the enabled + * flag into account. + */ + if (acpi_device_is_enabled(adev)) + return 0; + + /* Skip device that have not been enumerated. */ + if (!acpi_device_enumerated(adev)) { + dev_dbg(&adev->dev, "Still not enumerated\n"); + return 0; + } + } + + adev->flags.match_driver = false; + if (handler) { + if (handler->detach) + handler->detach(adev); + + adev->handler = NULL; + } else { + device_release_driver(&adev->dev); + } + /* + * Most likely, the device is going away, so put it into D3cold before + * that. + */ + acpi_device_set_power(adev, ACPI_STATE_D3_COLD); + adev->flags.initialized = false; + acpi_device_clear_enumerated(adev); + + return 0; +} + +static void acpi_scan_check_subtree(struct acpi_device *adev) +{ + acpi_scan_check_and_detach(adev, (void *)true); +} + static int acpi_scan_hot_remove(struct acpi_device *device) { acpi_handle handle = device->handle; @@ -289,75 +336,62 @@ static int acpi_scan_hot_remove(struct acpi_device *device) return 0; } -static int acpi_scan_device_not_enumerated(struct acpi_device *adev) +static int acpi_scan_rescan_bus(struct acpi_device *adev) { - if (!acpi_device_enumerated(adev)) { - dev_warn(&adev->dev, "Still not enumerated\n"); - return -EALREADY; - } - acpi_bus_trim(adev); - return 0; + struct acpi_scan_handler *handler = adev->handler; + int ret; + + if (handler && handler->hotplug.scan_dependent) + ret = handler->hotplug.scan_dependent(adev); + else + ret = acpi_bus_scan(adev->handle); + + if (ret) + dev_info(&adev->dev, "Namespace scan failure\n"); + + return ret; } static int acpi_scan_device_check(struct acpi_device *adev) { - int error; + struct acpi_device *parent; - acpi_bus_get_status(adev); - if (acpi_device_is_present(adev)) { - /* - * This function is only called for device objects for which - * matching scan handlers exist. The only situation in which - * the scan handler is not attached to this device object yet - * is when the device has just appeared (either it wasn't - * present at all before or it was removed and then added - * again). - */ - if (adev->handler) { - dev_warn(&adev->dev, "Already enumerated\n"); - return -EALREADY; - } - error = acpi_bus_scan(adev->handle); - if (error) { - dev_warn(&adev->dev, "Namespace scan failure\n"); - return error; - } - if (!adev->handler) { - dev_warn(&adev->dev, "Enumeration failure\n"); - error = -ENODEV; - } - } else { - error = acpi_scan_device_not_enumerated(adev); - } - return error; -} + acpi_scan_check_subtree(adev); -static int acpi_scan_bus_check(struct acpi_device *adev, void *not_used) -{ - struct acpi_scan_handler *handler = adev->handler; - int error; + if (!acpi_device_is_present(adev)) + return 0; - acpi_bus_get_status(adev); - if (!acpi_device_is_present(adev)) { - acpi_scan_device_not_enumerated(adev); + /* + * This function is only called for device objects for which matching + * scan handlers exist. The only situation in which the scan handler + * is not attached to this device object yet is when the device has + * just appeared (either it wasn't present at all before or it was + * removed and then added again). + */ + if (adev->handler) { + dev_dbg(&adev->dev, "Already enumerated\n"); return 0; } - if (handler && handler->hotplug.scan_dependent) - return handler->hotplug.scan_dependent(adev); - error = acpi_bus_scan(adev->handle); - if (error) { - dev_warn(&adev->dev, "Namespace scan failure\n"); - return error; - } - return acpi_dev_for_each_child(adev, acpi_scan_bus_check, NULL); + parent = acpi_dev_parent(adev); + if (!parent) + parent = adev; + + return acpi_scan_rescan_bus(parent); +} + +static int acpi_scan_bus_check(struct acpi_device *adev) +{ + acpi_scan_check_subtree(adev); + + return acpi_scan_rescan_bus(adev); } static int acpi_generic_hotplug_event(struct acpi_device *adev, u32 type) { switch (type) { case ACPI_NOTIFY_BUS_CHECK: - return acpi_scan_bus_check(adev, NULL); + return acpi_scan_bus_check(adev); case ACPI_NOTIFY_DEVICE_CHECK: return acpi_scan_device_check(adev); case ACPI_NOTIFY_EJECT_REQUEST: @@ -798,6 +832,7 @@ static const char * const acpi_honor_dep_ids[] = { "INTC1059", /* IVSC (TGL) driver must be loaded to allow i2c access to camera sensors */ "INTC1095", /* IVSC (ADL) driver must be loaded to allow i2c access to camera sensors */ "INTC100A", /* IVSC (RPL) driver must be loaded to allow i2c access to camera sensors */ + "INTC10CF", /* IVSC (MTL) driver must be loaded to allow i2c access to camera sensors */ NULL }; @@ -1922,6 +1957,11 @@ bool acpi_device_is_present(const struct acpi_device *adev) return adev->status.present || adev->status.functional; } +bool acpi_device_is_enabled(const struct acpi_device *adev) +{ + return adev->status.present && adev->status.enabled; +} + static bool acpi_scan_handler_matching(struct acpi_scan_handler *handler, const char *idstr, const struct acpi_device_id **matchid) @@ -2550,32 +2590,6 @@ int acpi_bus_scan(acpi_handle handle) } EXPORT_SYMBOL(acpi_bus_scan); -static int acpi_bus_trim_one(struct acpi_device *adev, void *not_used) -{ - struct acpi_scan_handler *handler = adev->handler; - - acpi_dev_for_each_child_reverse(adev, acpi_bus_trim_one, NULL); - - adev->flags.match_driver = false; - if (handler) { - if (handler->detach) - handler->detach(adev); - - adev->handler = NULL; - } else { - device_release_driver(&adev->dev); - } - /* - * Most likely, the device is going away, so put it into D3cold before - * that. - */ - acpi_device_set_power(adev, ACPI_STATE_D3_COLD); - adev->flags.initialized = false; - acpi_device_clear_enumerated(adev); - - return 0; -} - /** * acpi_bus_trim - Detach scan handlers and drivers from ACPI device objects. * @adev: Root of the ACPI namespace scope to walk. @@ -2584,7 +2598,7 @@ static int acpi_bus_trim_one(struct acpi_device *adev, void *not_used) */ void acpi_bus_trim(struct acpi_device *adev) { - acpi_bus_trim_one(adev, NULL); + acpi_scan_check_and_detach(adev, NULL); } EXPORT_SYMBOL_GPL(acpi_bus_trim); diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c index 4748e8061253..302dce0b2b50 100644 --- a/drivers/acpi/thermal.c +++ b/drivers/acpi/thermal.c @@ -47,6 +47,8 @@ #define ACPI_THERMAL_TRIP_PASSIVE (-1) +#define ACPI_THERMAL_MAX_NR_TRIPS (ACPI_THERMAL_MAX_ACTIVE + 3) + /* * This exception is thrown out in two cases: * 1.An invalid trip point becomes invalid or a valid trip point becomes invalid @@ -112,7 +114,6 @@ struct acpi_thermal { unsigned long polling_frequency; volatile u8 zombie; struct acpi_thermal_trips trips; - struct thermal_trip *trip_table; struct thermal_zone_device *thermal_zone; int kelvin_offset; /* in millidegrees */ struct work_struct thermal_check_work; @@ -451,26 +452,19 @@ fail: return false; } -static int acpi_thermal_get_trip_points(struct acpi_thermal *tz) +static void acpi_thermal_get_trip_points(struct acpi_thermal *tz) { - unsigned int count = 0; int i; - if (acpi_thermal_init_trip(tz, ACPI_THERMAL_TRIP_PASSIVE)) - count++; + acpi_thermal_init_trip(tz, ACPI_THERMAL_TRIP_PASSIVE); for (i = 0; i < ACPI_THERMAL_MAX_ACTIVE; i++) { - if (acpi_thermal_init_trip(tz, i)) - count++; - else + if (!acpi_thermal_init_trip(tz, i)) break; - } while (++i < ACPI_THERMAL_MAX_ACTIVE) tz->trips.active[i].trip.temp_dk = THERMAL_TEMP_INVALID; - - return count; } /* sys I/F for generic thermal sysfs support */ @@ -626,7 +620,7 @@ acpi_thermal_unbind_cooling_device(struct thermal_zone_device *thermal, return acpi_thermal_bind_unbind_cdev(thermal, cdev, false); } -static struct thermal_zone_device_ops acpi_thermal_zone_ops = { +static const struct thermal_zone_device_ops acpi_thermal_zone_ops = { .bind = acpi_thermal_bind_cooling_device, .unbind = acpi_thermal_unbind_cooling_device, .get_temp = thermal_get_temp, @@ -662,15 +656,16 @@ static void acpi_thermal_zone_sysfs_remove(struct acpi_thermal *tz) } static int acpi_thermal_register_thermal_zone(struct acpi_thermal *tz, + const struct thermal_trip *trip_table, unsigned int trip_count, int passive_delay) { int result; tz->thermal_zone = thermal_zone_device_register_with_trips("acpitz", - tz->trip_table, + trip_table, trip_count, - 0, tz, + tz, &acpi_thermal_zone_ops, NULL, passive_delay, @@ -823,10 +818,10 @@ static void acpi_thermal_free_thermal_zone(struct acpi_thermal *tz) static int acpi_thermal_add(struct acpi_device *device) { + struct thermal_trip trip_table[ACPI_THERMAL_MAX_NR_TRIPS] = { 0 }; struct acpi_thermal_trip *acpi_trip; struct thermal_trip *trip; struct acpi_thermal *tz; - unsigned int trip_count; int crit_temp, hot_temp; int passive_delay = 0; int result; @@ -848,21 +843,10 @@ static int acpi_thermal_add(struct acpi_device *device) acpi_thermal_aml_dependency_fix(tz); /* Get trip points [_CRT, _PSV, etc.] (required). */ - trip_count = acpi_thermal_get_trip_points(tz); + acpi_thermal_get_trip_points(tz); crit_temp = acpi_thermal_get_critical_trip(tz); - if (crit_temp != THERMAL_TEMP_INVALID) - trip_count++; - hot_temp = acpi_thermal_get_hot_trip(tz); - if (hot_temp != THERMAL_TEMP_INVALID) - trip_count++; - - if (!trip_count) { - pr_warn(FW_BUG "No valid trip points!\n"); - result = -ENODEV; - goto free_memory; - } /* Get temperature [_TMP] (required). */ result = acpi_thermal_get_temperature(tz); @@ -881,13 +865,7 @@ static int acpi_thermal_add(struct acpi_device *device) acpi_thermal_guess_offset(tz, crit_temp); - trip = kcalloc(trip_count, sizeof(*trip), GFP_KERNEL); - if (!trip) { - result = -ENOMEM; - goto free_memory; - } - - tz->trip_table = trip; + trip = trip_table; if (crit_temp != THERMAL_TEMP_INVALID) { trip->type = THERMAL_TRIP_CRITICAL; @@ -923,9 +901,17 @@ static int acpi_thermal_add(struct acpi_device *device) trip++; } - result = acpi_thermal_register_thermal_zone(tz, trip_count, passive_delay); + if (trip == trip_table) { + pr_warn(FW_BUG "No valid trip points!\n"); + result = -ENODEV; + goto free_memory; + } + + result = acpi_thermal_register_thermal_zone(tz, trip_table, + trip - trip_table, + passive_delay); if (result) - goto free_trips; + goto free_memory; refcount_set(&tz->thermal_check_count, 3); mutex_init(&tz->thermal_check_lock); @@ -944,8 +930,6 @@ static int acpi_thermal_add(struct acpi_device *device) flush_wq: flush_workqueue(acpi_thermal_pm_queue); acpi_thermal_unregister_thermal_zone(tz); -free_trips: - kfree(tz->trip_table); free_memory: acpi_thermal_free_thermal_zone(tz); @@ -966,7 +950,6 @@ static void acpi_thermal_remove(struct acpi_device *device) flush_workqueue(acpi_thermal_pm_queue); acpi_thermal_unregister_thermal_zone(tz); - kfree(tz->trip_table); acpi_thermal_free_thermal_zone(tz); } diff --git a/drivers/acpi/thermal_lib.c b/drivers/acpi/thermal_lib.c index 4e0519ca9739..6214d6ebe1fa 100644 --- a/drivers/acpi/thermal_lib.c +++ b/drivers/acpi/thermal_lib.c @@ -100,7 +100,7 @@ static int thermal_temp(int error, int temp_decik, int *ret_temp) */ int thermal_acpi_active_trip_temp(struct acpi_device *adev, int id, int *ret_temp) { - int temp_decik; + int temp_decik = 0; int ret = acpi_active_trip_temp(adev, id, &temp_decik); return thermal_temp(ret, temp_decik, ret_temp); @@ -119,7 +119,7 @@ EXPORT_SYMBOL_GPL(thermal_acpi_active_trip_temp); */ int thermal_acpi_passive_trip_temp(struct acpi_device *adev, int *ret_temp) { - int temp_decik; + int temp_decik = 0; int ret = acpi_passive_trip_temp(adev, &temp_decik); return thermal_temp(ret, temp_decik, ret_temp); @@ -139,7 +139,7 @@ EXPORT_SYMBOL_GPL(thermal_acpi_passive_trip_temp); */ int thermal_acpi_hot_trip_temp(struct acpi_device *adev, int *ret_temp) { - int temp_decik; + int temp_decik = 0; int ret = acpi_hot_trip_temp(adev, &temp_decik); return thermal_temp(ret, temp_decik, ret_temp); @@ -158,7 +158,7 @@ EXPORT_SYMBOL_GPL(thermal_acpi_hot_trip_temp); */ int thermal_acpi_critical_trip_temp(struct acpi_device *adev, int *ret_temp) { - int temp_decik; + int temp_decik = 0; int ret = acpi_critical_trip_temp(adev, &temp_decik); return thermal_temp(ret, temp_decik, ret_temp); diff --git a/drivers/acpi/utils.c b/drivers/acpi/utils.c index abac5cc25477..202234ba54bd 100644 --- a/drivers/acpi/utils.c +++ b/drivers/acpi/utils.c @@ -559,7 +559,7 @@ EXPORT_SYMBOL(acpi_evaluate_ost); * * Caller must free the returned buffer */ -static char *acpi_handle_path(acpi_handle handle) +char *acpi_handle_path(acpi_handle handle) { struct acpi_buffer buffer = { .length = ACPI_ALLOCATE_BUFFER, diff --git a/drivers/acpi/x86/s2idle.c b/drivers/acpi/x86/s2idle.c index 7d64e655f1b8..cd84af23f7ea 100644 --- a/drivers/acpi/x86/s2idle.c +++ b/drivers/acpi/x86/s2idle.c @@ -488,7 +488,21 @@ static int lps0_device_attach(struct acpi_device *adev, rev_id = 1; lps0_dsm_func_mask = validate_dsm(adev->handle, ACPI_LPS0_DSM_UUID, rev_id, &lps0_dsm_guid); - lps0_dsm_func_mask_microsoft = -EINVAL; + if (lps0_dsm_func_mask > 0 && lps0_dsm_func_mask_microsoft > 0) { + unsigned int func_mask; + + /* + * Avoid evaluating the same _DSM function for two + * different UUIDs and prioritize the MSFT one. + */ + func_mask = lps0_dsm_func_mask & lps0_dsm_func_mask_microsoft; + if (func_mask) { + acpi_handle_info(adev->handle, + "Duplicate LPS0 _DSM functions (mask: 0x%x)\n", + func_mask); + lps0_dsm_func_mask &= ~func_mask; + } + } } if (lps0_dsm_func_mask < 0 && lps0_dsm_func_mask_microsoft < 0) @@ -549,19 +563,22 @@ int acpi_s2idle_prepare_late(void) lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); /* LPS0 entry */ - if (lps0_dsm_func_mask > 0) - acpi_sleep_run_lps0_dsm(acpi_s2idle_vendor_amd() ? - ACPI_LPS0_ENTRY_AMD : - ACPI_LPS0_ENTRY, + if (lps0_dsm_func_mask > 0 && acpi_s2idle_vendor_amd()) + acpi_sleep_run_lps0_dsm(ACPI_LPS0_ENTRY_AMD, lps0_dsm_func_mask, lps0_dsm_guid); + if (lps0_dsm_func_mask_microsoft > 0) { - /* modern standby entry */ + /* Modern Standby entry */ acpi_sleep_run_lps0_dsm(ACPI_LPS0_MS_ENTRY, lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); acpi_sleep_run_lps0_dsm(ACPI_LPS0_ENTRY, lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); } + if (lps0_dsm_func_mask > 0 && !acpi_s2idle_vendor_amd()) + acpi_sleep_run_lps0_dsm(ACPI_LPS0_ENTRY, + lps0_dsm_func_mask, lps0_dsm_guid); + list_for_each_entry(handler, &lps0_s2idle_devops_head, list_node) { if (handler->prepare) handler->prepare(); @@ -600,14 +617,14 @@ void acpi_s2idle_restore_early(void) ACPI_LPS0_EXIT_AMD : ACPI_LPS0_EXIT, lps0_dsm_func_mask, lps0_dsm_guid); - if (lps0_dsm_func_mask_microsoft > 0) + + if (lps0_dsm_func_mask_microsoft > 0) { acpi_sleep_run_lps0_dsm(ACPI_LPS0_EXIT, lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); - - /* Modern standby exit */ - if (lps0_dsm_func_mask_microsoft > 0) + /* Modern Standby exit */ acpi_sleep_run_lps0_dsm(ACPI_LPS0_MS_EXIT, lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); + } /* Screen on */ if (lps0_dsm_func_mask_microsoft > 0) diff --git a/drivers/acpi/x86/utils.c b/drivers/acpi/x86/utils.c index bc65ebfcdf76..90c3d2eab9e9 100644 --- a/drivers/acpi/x86/utils.c +++ b/drivers/acpi/x86/utils.c @@ -428,7 +428,7 @@ bool acpi_quirk_skip_i2c_client_enumeration(struct acpi_device *adev) } EXPORT_SYMBOL_GPL(acpi_quirk_skip_i2c_client_enumeration); -int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *skip) +static int acpi_dmi_skip_serdev_enumeration(struct device *controller_parent, bool *skip) { struct acpi_device *adev = ACPI_COMPANION(controller_parent); const struct dmi_system_id *dmi_id; @@ -436,8 +436,6 @@ int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *s u64 uid; int ret; - *skip = false; - ret = acpi_dev_uid_to_integer(adev, &uid); if (ret) return 0; @@ -463,7 +461,6 @@ int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *s return 0; } -EXPORT_SYMBOL_GPL(acpi_quirk_skip_serdev_enumeration); bool acpi_quirk_skip_gpio_event_handlers(void) { @@ -478,8 +475,41 @@ bool acpi_quirk_skip_gpio_event_handlers(void) return (quirks & ACPI_QUIRK_SKIP_GPIO_EVENT_HANDLERS); } EXPORT_SYMBOL_GPL(acpi_quirk_skip_gpio_event_handlers); +#else +static int acpi_dmi_skip_serdev_enumeration(struct device *controller_parent, bool *skip) +{ + return 0; +} #endif +int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *skip) +{ + struct acpi_device *adev = ACPI_COMPANION(controller_parent); + + *skip = false; + + /* + * The DELL0501 ACPI HID represents an UART (CID is set to PNP0501) with + * a backlight-controller attached. There is no separate ACPI device with + * an UartSerialBusV2() resource to model the backlight-controller. + * Set skip to true so that the tty core creates a serdev ctrl device. + * The backlight driver will manually create the serdev client device. + */ + if (acpi_dev_hid_match(adev, "DELL0501")) { + *skip = true; + /* + * Create a platform dev for dell-uart-backlight to bind to. + * This is a static device, so no need to store the result. + */ + platform_device_register_simple("dell-uart-backlight", PLATFORM_DEVID_NONE, + NULL, 0); + return 0; + } + + return acpi_dmi_skip_serdev_enumeration(controller_parent, skip); +} +EXPORT_SYMBOL_GPL(acpi_quirk_skip_serdev_enumeration); + /* Lists of PMIC ACPI HIDs with an (often better) native charger driver */ static const struct { const char *hid; diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig index 42b51c9812a0..928ec93c6b45 100644 --- a/drivers/ata/Kconfig +++ b/drivers/ata/Kconfig @@ -116,15 +116,14 @@ config SATA_AHCI If unsure, say N. config SATA_MOBILE_LPM_POLICY - int "Default SATA Link Power Management policy for low power chipsets" + int "Default SATA Link Power Management policy" range 0 4 default 0 depends on SATA_AHCI help Select the Default SATA Link Power Management (LPM) policy to use for chipsets / "South Bridges" supporting low-power modes. Such - chipsets are typically found on most laptops but desktops and - servers now also widely use chipsets supporting low power modes. + chipsets are ubiquitous across laptops, desktops and servers. The value set has the following meanings: 0 => Keep firmware settings diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 682ff550ccfb..78570684ff68 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -50,11 +50,18 @@ enum board_ids { board_ahci, board_ahci_43bit_dma, board_ahci_ign_iferr, - board_ahci_low_power, board_ahci_no_debounce_delay, - board_ahci_nomsi, - board_ahci_noncq, - board_ahci_nosntf, + board_ahci_no_msi, + /* + * board_ahci_pcs_quirk is for legacy Intel platforms. + * Modern Intel platforms should use board_ahci instead. + * (Some modern Intel platforms might have been added with + * board_ahci_pcs_quirk, however, we cannot change them to board_ahci + * without testing that the platform actually works without the quirk.) + */ + board_ahci_pcs_quirk, + board_ahci_pcs_quirk_no_devslp, + board_ahci_pcs_quirk_no_sntf, board_ahci_yes_fbs, /* board IDs for specific chipsets in alphabetical order */ @@ -68,12 +75,6 @@ enum board_ids { board_ahci_sb700, /* for SB700 and SB800 */ board_ahci_vt8251, - /* - * board IDs for Intel chipsets that support more than 6 ports - * *and* end up needing the PCS quirk. - */ - board_ahci_pcs7, - /* aliases */ board_ahci_mcp_linux = board_ahci_mcp65, board_ahci_mcp67 = board_ahci_mcp65, @@ -143,13 +144,6 @@ static const struct ata_port_info ahci_port_info[] = { .udma_mask = ATA_UDMA6, .port_ops = &ahci_ops, }, - [board_ahci_low_power] = { - AHCI_HFLAGS (AHCI_HFLAG_USE_LPM_POLICY), - .flags = AHCI_FLAG_COMMON, - .pio_mask = ATA_PIO4, - .udma_mask = ATA_UDMA6, - .port_ops = &ahci_ops, - }, [board_ahci_no_debounce_delay] = { .flags = AHCI_FLAG_COMMON, .link_flags = ATA_LFLAG_NO_DEBOUNCE_DELAY, @@ -157,22 +151,31 @@ static const struct ata_port_info ahci_port_info[] = { .udma_mask = ATA_UDMA6, .port_ops = &ahci_ops, }, - [board_ahci_nomsi] = { + [board_ahci_no_msi] = { AHCI_HFLAGS (AHCI_HFLAG_NO_MSI), .flags = AHCI_FLAG_COMMON, .pio_mask = ATA_PIO4, .udma_mask = ATA_UDMA6, .port_ops = &ahci_ops, }, - [board_ahci_noncq] = { - AHCI_HFLAGS (AHCI_HFLAG_NO_NCQ), + [board_ahci_pcs_quirk] = { + AHCI_HFLAGS (AHCI_HFLAG_INTEL_PCS_QUIRK), .flags = AHCI_FLAG_COMMON, .pio_mask = ATA_PIO4, .udma_mask = ATA_UDMA6, .port_ops = &ahci_ops, }, - [board_ahci_nosntf] = { - AHCI_HFLAGS (AHCI_HFLAG_NO_SNTF), + [board_ahci_pcs_quirk_no_devslp] = { + AHCI_HFLAGS (AHCI_HFLAG_INTEL_PCS_QUIRK | + AHCI_HFLAG_NO_DEVSLP), + .flags = AHCI_FLAG_COMMON, + .pio_mask = ATA_PIO4, + .udma_mask = ATA_UDMA6, + .port_ops = &ahci_ops, + }, + [board_ahci_pcs_quirk_no_sntf] = { + AHCI_HFLAGS (AHCI_HFLAG_INTEL_PCS_QUIRK | + AHCI_HFLAG_NO_SNTF), .flags = AHCI_FLAG_COMMON, .pio_mask = ATA_PIO4, .udma_mask = ATA_UDMA6, @@ -194,6 +197,7 @@ static const struct ata_port_info ahci_port_info[] = { .port_ops = &ahci_ops, }, [board_ahci_avn] = { + AHCI_HFLAGS (AHCI_HFLAG_INTEL_PCS_QUIRK), .flags = AHCI_FLAG_COMMON, .pio_mask = ATA_PIO4, .udma_mask = ATA_UDMA6, @@ -252,119 +256,113 @@ static const struct ata_port_info ahci_port_info[] = { .udma_mask = ATA_UDMA6, .port_ops = &ahci_vt8251_ops, }, - [board_ahci_pcs7] = { - .flags = AHCI_FLAG_COMMON, - .pio_mask = ATA_PIO4, - .udma_mask = ATA_UDMA6, - .port_ops = &ahci_ops, - }, }; static const struct pci_device_id ahci_pci_tbl[] = { /* Intel */ - { PCI_VDEVICE(INTEL, 0x06d6), board_ahci }, /* Comet Lake PCH-H RAID */ - { PCI_VDEVICE(INTEL, 0x2652), board_ahci }, /* ICH6 */ - { PCI_VDEVICE(INTEL, 0x2653), board_ahci }, /* ICH6M */ - { PCI_VDEVICE(INTEL, 0x27c1), board_ahci }, /* ICH7 */ - { PCI_VDEVICE(INTEL, 0x27c5), board_ahci }, /* ICH7M */ - { PCI_VDEVICE(INTEL, 0x27c3), board_ahci }, /* ICH7R */ + { PCI_VDEVICE(INTEL, 0x06d6), board_ahci_pcs_quirk }, /* Comet Lake PCH-H RAID */ + { PCI_VDEVICE(INTEL, 0x2652), board_ahci_pcs_quirk }, /* ICH6 */ + { PCI_VDEVICE(INTEL, 0x2653), board_ahci_pcs_quirk }, /* ICH6M */ + { PCI_VDEVICE(INTEL, 0x27c1), board_ahci_pcs_quirk }, /* ICH7 */ + { PCI_VDEVICE(INTEL, 0x27c5), board_ahci_pcs_quirk }, /* ICH7M */ + { PCI_VDEVICE(INTEL, 0x27c3), board_ahci_pcs_quirk }, /* ICH7R */ { PCI_VDEVICE(AL, 0x5288), board_ahci_ign_iferr }, /* ULi M5288 */ - { PCI_VDEVICE(INTEL, 0x2681), board_ahci }, /* ESB2 */ - { PCI_VDEVICE(INTEL, 0x2682), board_ahci }, /* ESB2 */ - { PCI_VDEVICE(INTEL, 0x2683), board_ahci }, /* ESB2 */ - { PCI_VDEVICE(INTEL, 0x27c6), board_ahci }, /* ICH7-M DH */ - { PCI_VDEVICE(INTEL, 0x2821), board_ahci }, /* ICH8 */ - { PCI_VDEVICE(INTEL, 0x2822), board_ahci_nosntf }, /* ICH8/Lewisburg RAID*/ - { PCI_VDEVICE(INTEL, 0x2824), board_ahci }, /* ICH8 */ - { PCI_VDEVICE(INTEL, 0x2829), board_ahci }, /* ICH8M */ - { PCI_VDEVICE(INTEL, 0x282a), board_ahci }, /* ICH8M */ - { PCI_VDEVICE(INTEL, 0x2922), board_ahci }, /* ICH9 */ - { PCI_VDEVICE(INTEL, 0x2923), board_ahci }, /* ICH9 */ - { PCI_VDEVICE(INTEL, 0x2924), board_ahci }, /* ICH9 */ - { PCI_VDEVICE(INTEL, 0x2925), board_ahci }, /* ICH9 */ - { PCI_VDEVICE(INTEL, 0x2927), board_ahci }, /* ICH9 */ - { PCI_VDEVICE(INTEL, 0x2929), board_ahci_low_power }, /* ICH9M */ - { PCI_VDEVICE(INTEL, 0x292a), board_ahci_low_power }, /* ICH9M */ - { PCI_VDEVICE(INTEL, 0x292b), board_ahci_low_power }, /* ICH9M */ - { PCI_VDEVICE(INTEL, 0x292c), board_ahci_low_power }, /* ICH9M */ - { PCI_VDEVICE(INTEL, 0x292f), board_ahci_low_power }, /* ICH9M */ - { PCI_VDEVICE(INTEL, 0x294d), board_ahci }, /* ICH9 */ - { PCI_VDEVICE(INTEL, 0x294e), board_ahci_low_power }, /* ICH9M */ - { PCI_VDEVICE(INTEL, 0x502a), board_ahci }, /* Tolapai */ - { PCI_VDEVICE(INTEL, 0x502b), board_ahci }, /* Tolapai */ - { PCI_VDEVICE(INTEL, 0x3a05), board_ahci }, /* ICH10 */ - { PCI_VDEVICE(INTEL, 0x3a22), board_ahci }, /* ICH10 */ - { PCI_VDEVICE(INTEL, 0x3a25), board_ahci }, /* ICH10 */ - { PCI_VDEVICE(INTEL, 0x3b22), board_ahci }, /* PCH AHCI */ - { PCI_VDEVICE(INTEL, 0x3b23), board_ahci }, /* PCH AHCI */ - { PCI_VDEVICE(INTEL, 0x3b24), board_ahci }, /* PCH RAID */ - { PCI_VDEVICE(INTEL, 0x3b25), board_ahci }, /* PCH RAID */ - { PCI_VDEVICE(INTEL, 0x3b29), board_ahci_low_power }, /* PCH M AHCI */ - { PCI_VDEVICE(INTEL, 0x3b2b), board_ahci }, /* PCH RAID */ - { PCI_VDEVICE(INTEL, 0x3b2c), board_ahci_low_power }, /* PCH M RAID */ - { PCI_VDEVICE(INTEL, 0x3b2f), board_ahci }, /* PCH AHCI */ - { PCI_VDEVICE(INTEL, 0x19b0), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19b1), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19b2), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19b3), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19b4), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19b5), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19b6), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19b7), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19bE), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19bF), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19c0), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19c1), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19c2), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19c3), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19c4), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19c5), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19c6), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19c7), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19cE), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x19cF), board_ahci_pcs7 }, /* DNV AHCI */ - { PCI_VDEVICE(INTEL, 0x1c02), board_ahci }, /* CPT AHCI */ - { PCI_VDEVICE(INTEL, 0x1c03), board_ahci_low_power }, /* CPT M AHCI */ - { PCI_VDEVICE(INTEL, 0x1c04), board_ahci }, /* CPT RAID */ - { PCI_VDEVICE(INTEL, 0x1c05), board_ahci_low_power }, /* CPT M RAID */ - { PCI_VDEVICE(INTEL, 0x1c06), board_ahci }, /* CPT RAID */ - { PCI_VDEVICE(INTEL, 0x1c07), board_ahci }, /* CPT RAID */ - { PCI_VDEVICE(INTEL, 0x1d02), board_ahci }, /* PBG AHCI */ - { PCI_VDEVICE(INTEL, 0x1d04), board_ahci }, /* PBG RAID */ - { PCI_VDEVICE(INTEL, 0x1d06), board_ahci }, /* PBG RAID */ - { PCI_VDEVICE(INTEL, 0x2323), board_ahci }, /* DH89xxCC AHCI */ - { PCI_VDEVICE(INTEL, 0x1e02), board_ahci }, /* Panther Point AHCI */ - { PCI_VDEVICE(INTEL, 0x1e03), board_ahci_low_power }, /* Panther M AHCI */ - { PCI_VDEVICE(INTEL, 0x1e04), board_ahci }, /* Panther Point RAID */ - { PCI_VDEVICE(INTEL, 0x1e05), board_ahci }, /* Panther Point RAID */ - { PCI_VDEVICE(INTEL, 0x1e06), board_ahci }, /* Panther Point RAID */ - { PCI_VDEVICE(INTEL, 0x1e07), board_ahci_low_power }, /* Panther M RAID */ - { PCI_VDEVICE(INTEL, 0x1e0e), board_ahci }, /* Panther Point RAID */ - { PCI_VDEVICE(INTEL, 0x8c02), board_ahci }, /* Lynx Point AHCI */ - { PCI_VDEVICE(INTEL, 0x8c03), board_ahci_low_power }, /* Lynx M AHCI */ - { PCI_VDEVICE(INTEL, 0x8c04), board_ahci }, /* Lynx Point RAID */ - { PCI_VDEVICE(INTEL, 0x8c05), board_ahci_low_power }, /* Lynx M RAID */ - { PCI_VDEVICE(INTEL, 0x8c06), board_ahci }, /* Lynx Point RAID */ - { PCI_VDEVICE(INTEL, 0x8c07), board_ahci_low_power }, /* Lynx M RAID */ - { PCI_VDEVICE(INTEL, 0x8c0e), board_ahci }, /* Lynx Point RAID */ - { PCI_VDEVICE(INTEL, 0x8c0f), board_ahci_low_power }, /* Lynx M RAID */ - { PCI_VDEVICE(INTEL, 0x9c02), board_ahci_low_power }, /* Lynx LP AHCI */ - { PCI_VDEVICE(INTEL, 0x9c03), board_ahci_low_power }, /* Lynx LP AHCI */ - { PCI_VDEVICE(INTEL, 0x9c04), board_ahci_low_power }, /* Lynx LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c05), board_ahci_low_power }, /* Lynx LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c06), board_ahci_low_power }, /* Lynx LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c07), board_ahci_low_power }, /* Lynx LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c0e), board_ahci_low_power }, /* Lynx LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c0f), board_ahci_low_power }, /* Lynx LP RAID */ - { PCI_VDEVICE(INTEL, 0x9dd3), board_ahci_low_power }, /* Cannon Lake PCH-LP AHCI */ - { PCI_VDEVICE(INTEL, 0x1f22), board_ahci }, /* Avoton AHCI */ - { PCI_VDEVICE(INTEL, 0x1f23), board_ahci }, /* Avoton AHCI */ - { PCI_VDEVICE(INTEL, 0x1f24), board_ahci }, /* Avoton RAID */ - { PCI_VDEVICE(INTEL, 0x1f25), board_ahci }, /* Avoton RAID */ - { PCI_VDEVICE(INTEL, 0x1f26), board_ahci }, /* Avoton RAID */ - { PCI_VDEVICE(INTEL, 0x1f27), board_ahci }, /* Avoton RAID */ - { PCI_VDEVICE(INTEL, 0x1f2e), board_ahci }, /* Avoton RAID */ - { PCI_VDEVICE(INTEL, 0x1f2f), board_ahci }, /* Avoton RAID */ + { PCI_VDEVICE(INTEL, 0x2681), board_ahci_pcs_quirk }, /* ESB2 */ + { PCI_VDEVICE(INTEL, 0x2682), board_ahci_pcs_quirk }, /* ESB2 */ + { PCI_VDEVICE(INTEL, 0x2683), board_ahci_pcs_quirk }, /* ESB2 */ + { PCI_VDEVICE(INTEL, 0x27c6), board_ahci_pcs_quirk }, /* ICH7-M DH */ + { PCI_VDEVICE(INTEL, 0x2821), board_ahci_pcs_quirk }, /* ICH8 */ + { PCI_VDEVICE(INTEL, 0x2822), board_ahci_pcs_quirk_no_sntf }, /* ICH8/Lewisburg RAID*/ + { PCI_VDEVICE(INTEL, 0x2824), board_ahci_pcs_quirk }, /* ICH8 */ + { PCI_VDEVICE(INTEL, 0x2829), board_ahci_pcs_quirk }, /* ICH8M */ + { PCI_VDEVICE(INTEL, 0x282a), board_ahci_pcs_quirk }, /* ICH8M */ + { PCI_VDEVICE(INTEL, 0x2922), board_ahci_pcs_quirk }, /* ICH9 */ + { PCI_VDEVICE(INTEL, 0x2923), board_ahci_pcs_quirk }, /* ICH9 */ + { PCI_VDEVICE(INTEL, 0x2924), board_ahci_pcs_quirk }, /* ICH9 */ + { PCI_VDEVICE(INTEL, 0x2925), board_ahci_pcs_quirk }, /* ICH9 */ + { PCI_VDEVICE(INTEL, 0x2927), board_ahci_pcs_quirk }, /* ICH9 */ + { PCI_VDEVICE(INTEL, 0x2929), board_ahci_pcs_quirk }, /* ICH9M */ + { PCI_VDEVICE(INTEL, 0x292a), board_ahci_pcs_quirk }, /* ICH9M */ + { PCI_VDEVICE(INTEL, 0x292b), board_ahci_pcs_quirk }, /* ICH9M */ + { PCI_VDEVICE(INTEL, 0x292c), board_ahci_pcs_quirk }, /* ICH9M */ + { PCI_VDEVICE(INTEL, 0x292f), board_ahci_pcs_quirk }, /* ICH9M */ + { PCI_VDEVICE(INTEL, 0x294d), board_ahci_pcs_quirk }, /* ICH9 */ + { PCI_VDEVICE(INTEL, 0x294e), board_ahci_pcs_quirk }, /* ICH9M */ + { PCI_VDEVICE(INTEL, 0x502a), board_ahci_pcs_quirk }, /* Tolapai */ + { PCI_VDEVICE(INTEL, 0x502b), board_ahci_pcs_quirk }, /* Tolapai */ + { PCI_VDEVICE(INTEL, 0x3a05), board_ahci_pcs_quirk }, /* ICH10 */ + { PCI_VDEVICE(INTEL, 0x3a22), board_ahci_pcs_quirk }, /* ICH10 */ + { PCI_VDEVICE(INTEL, 0x3a25), board_ahci_pcs_quirk }, /* ICH10 */ + { PCI_VDEVICE(INTEL, 0x3b22), board_ahci_pcs_quirk }, /* PCH AHCI */ + { PCI_VDEVICE(INTEL, 0x3b23), board_ahci_pcs_quirk }, /* PCH AHCI */ + { PCI_VDEVICE(INTEL, 0x3b24), board_ahci_pcs_quirk }, /* PCH RAID */ + { PCI_VDEVICE(INTEL, 0x3b25), board_ahci_pcs_quirk }, /* PCH RAID */ + { PCI_VDEVICE(INTEL, 0x3b29), board_ahci_pcs_quirk }, /* PCH M AHCI */ + { PCI_VDEVICE(INTEL, 0x3b2b), board_ahci_pcs_quirk }, /* PCH RAID */ + { PCI_VDEVICE(INTEL, 0x3b2c), board_ahci_pcs_quirk }, /* PCH M RAID */ + { PCI_VDEVICE(INTEL, 0x3b2f), board_ahci_pcs_quirk }, /* PCH AHCI */ + { PCI_VDEVICE(INTEL, 0x19b0), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19b1), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19b2), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19b3), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19b4), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19b5), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19b6), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19b7), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19bE), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19bF), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19c0), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19c1), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19c2), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19c3), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19c4), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19c5), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19c6), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19c7), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19cE), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x19cF), board_ahci }, /* DNV AHCI */ + { PCI_VDEVICE(INTEL, 0x1c02), board_ahci_pcs_quirk }, /* CPT AHCI */ + { PCI_VDEVICE(INTEL, 0x1c03), board_ahci_pcs_quirk }, /* CPT M AHCI */ + { PCI_VDEVICE(INTEL, 0x1c04), board_ahci_pcs_quirk }, /* CPT RAID */ + { PCI_VDEVICE(INTEL, 0x1c05), board_ahci_pcs_quirk }, /* CPT M RAID */ + { PCI_VDEVICE(INTEL, 0x1c06), board_ahci_pcs_quirk }, /* CPT RAID */ + { PCI_VDEVICE(INTEL, 0x1c07), board_ahci_pcs_quirk }, /* CPT RAID */ + { PCI_VDEVICE(INTEL, 0x1d02), board_ahci_pcs_quirk }, /* PBG AHCI */ + { PCI_VDEVICE(INTEL, 0x1d04), board_ahci_pcs_quirk }, /* PBG RAID */ + { PCI_VDEVICE(INTEL, 0x1d06), board_ahci_pcs_quirk }, /* PBG RAID */ + { PCI_VDEVICE(INTEL, 0x2323), board_ahci_pcs_quirk }, /* DH89xxCC AHCI */ + { PCI_VDEVICE(INTEL, 0x1e02), board_ahci_pcs_quirk }, /* Panther Point AHCI */ + { PCI_VDEVICE(INTEL, 0x1e03), board_ahci_pcs_quirk }, /* Panther M AHCI */ + { PCI_VDEVICE(INTEL, 0x1e04), board_ahci_pcs_quirk }, /* Panther Point RAID */ + { PCI_VDEVICE(INTEL, 0x1e05), board_ahci_pcs_quirk }, /* Panther Point RAID */ + { PCI_VDEVICE(INTEL, 0x1e06), board_ahci_pcs_quirk }, /* Panther Point RAID */ + { PCI_VDEVICE(INTEL, 0x1e07), board_ahci_pcs_quirk }, /* Panther M RAID */ + { PCI_VDEVICE(INTEL, 0x1e0e), board_ahci_pcs_quirk }, /* Panther Point RAID */ + { PCI_VDEVICE(INTEL, 0x8c02), board_ahci_pcs_quirk }, /* Lynx Point AHCI */ + { PCI_VDEVICE(INTEL, 0x8c03), board_ahci_pcs_quirk }, /* Lynx M AHCI */ + { PCI_VDEVICE(INTEL, 0x8c04), board_ahci_pcs_quirk }, /* Lynx Point RAID */ + { PCI_VDEVICE(INTEL, 0x8c05), board_ahci_pcs_quirk }, /* Lynx M RAID */ + { PCI_VDEVICE(INTEL, 0x8c06), board_ahci_pcs_quirk }, /* Lynx Point RAID */ + { PCI_VDEVICE(INTEL, 0x8c07), board_ahci_pcs_quirk }, /* Lynx M RAID */ + { PCI_VDEVICE(INTEL, 0x8c0e), board_ahci_pcs_quirk }, /* Lynx Point RAID */ + { PCI_VDEVICE(INTEL, 0x8c0f), board_ahci_pcs_quirk }, /* Lynx M RAID */ + { PCI_VDEVICE(INTEL, 0x9c02), board_ahci_pcs_quirk }, /* Lynx LP AHCI */ + { PCI_VDEVICE(INTEL, 0x9c03), board_ahci_pcs_quirk }, /* Lynx LP AHCI */ + { PCI_VDEVICE(INTEL, 0x9c04), board_ahci_pcs_quirk }, /* Lynx LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c05), board_ahci_pcs_quirk }, /* Lynx LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c06), board_ahci_pcs_quirk }, /* Lynx LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c07), board_ahci_pcs_quirk }, /* Lynx LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c0e), board_ahci_pcs_quirk }, /* Lynx LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c0f), board_ahci_pcs_quirk }, /* Lynx LP RAID */ + { PCI_VDEVICE(INTEL, 0x9dd3), board_ahci_pcs_quirk }, /* Cannon Lake PCH-LP AHCI */ + { PCI_VDEVICE(INTEL, 0x1f22), board_ahci_pcs_quirk }, /* Avoton AHCI */ + { PCI_VDEVICE(INTEL, 0x1f23), board_ahci_pcs_quirk }, /* Avoton AHCI */ + { PCI_VDEVICE(INTEL, 0x1f24), board_ahci_pcs_quirk }, /* Avoton RAID */ + { PCI_VDEVICE(INTEL, 0x1f25), board_ahci_pcs_quirk }, /* Avoton RAID */ + { PCI_VDEVICE(INTEL, 0x1f26), board_ahci_pcs_quirk }, /* Avoton RAID */ + { PCI_VDEVICE(INTEL, 0x1f27), board_ahci_pcs_quirk }, /* Avoton RAID */ + { PCI_VDEVICE(INTEL, 0x1f2e), board_ahci_pcs_quirk }, /* Avoton RAID */ + { PCI_VDEVICE(INTEL, 0x1f2f), board_ahci_pcs_quirk }, /* Avoton RAID */ { PCI_VDEVICE(INTEL, 0x1f32), board_ahci_avn }, /* Avoton AHCI */ { PCI_VDEVICE(INTEL, 0x1f33), board_ahci_avn }, /* Avoton AHCI */ { PCI_VDEVICE(INTEL, 0x1f34), board_ahci_avn }, /* Avoton RAID */ @@ -373,65 +371,65 @@ static const struct pci_device_id ahci_pci_tbl[] = { { PCI_VDEVICE(INTEL, 0x1f37), board_ahci_avn }, /* Avoton RAID */ { PCI_VDEVICE(INTEL, 0x1f3e), board_ahci_avn }, /* Avoton RAID */ { PCI_VDEVICE(INTEL, 0x1f3f), board_ahci_avn }, /* Avoton RAID */ - { PCI_VDEVICE(INTEL, 0x2823), board_ahci }, /* Wellsburg/Lewisburg AHCI*/ - { PCI_VDEVICE(INTEL, 0x2826), board_ahci }, /* *burg SATA0 'RAID' */ - { PCI_VDEVICE(INTEL, 0x2827), board_ahci }, /* *burg SATA1 'RAID' */ - { PCI_VDEVICE(INTEL, 0x282f), board_ahci }, /* *burg SATA2 'RAID' */ - { PCI_VDEVICE(INTEL, 0x43d4), board_ahci }, /* Rocket Lake PCH-H RAID */ - { PCI_VDEVICE(INTEL, 0x43d5), board_ahci }, /* Rocket Lake PCH-H RAID */ - { PCI_VDEVICE(INTEL, 0x43d6), board_ahci }, /* Rocket Lake PCH-H RAID */ - { PCI_VDEVICE(INTEL, 0x43d7), board_ahci }, /* Rocket Lake PCH-H RAID */ - { PCI_VDEVICE(INTEL, 0x8d02), board_ahci }, /* Wellsburg AHCI */ - { PCI_VDEVICE(INTEL, 0x8d04), board_ahci }, /* Wellsburg RAID */ - { PCI_VDEVICE(INTEL, 0x8d06), board_ahci }, /* Wellsburg RAID */ - { PCI_VDEVICE(INTEL, 0x8d0e), board_ahci }, /* Wellsburg RAID */ - { PCI_VDEVICE(INTEL, 0x8d62), board_ahci }, /* Wellsburg AHCI */ - { PCI_VDEVICE(INTEL, 0x8d64), board_ahci }, /* Wellsburg RAID */ - { PCI_VDEVICE(INTEL, 0x8d66), board_ahci }, /* Wellsburg RAID */ - { PCI_VDEVICE(INTEL, 0x8d6e), board_ahci }, /* Wellsburg RAID */ - { PCI_VDEVICE(INTEL, 0x23a3), board_ahci }, /* Coleto Creek AHCI */ - { PCI_VDEVICE(INTEL, 0x9c83), board_ahci_low_power }, /* Wildcat LP AHCI */ - { PCI_VDEVICE(INTEL, 0x9c85), board_ahci_low_power }, /* Wildcat LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c87), board_ahci_low_power }, /* Wildcat LP RAID */ - { PCI_VDEVICE(INTEL, 0x9c8f), board_ahci_low_power }, /* Wildcat LP RAID */ - { PCI_VDEVICE(INTEL, 0x8c82), board_ahci }, /* 9 Series AHCI */ - { PCI_VDEVICE(INTEL, 0x8c83), board_ahci_low_power }, /* 9 Series M AHCI */ - { PCI_VDEVICE(INTEL, 0x8c84), board_ahci }, /* 9 Series RAID */ - { PCI_VDEVICE(INTEL, 0x8c85), board_ahci_low_power }, /* 9 Series M RAID */ - { PCI_VDEVICE(INTEL, 0x8c86), board_ahci }, /* 9 Series RAID */ - { PCI_VDEVICE(INTEL, 0x8c87), board_ahci_low_power }, /* 9 Series M RAID */ - { PCI_VDEVICE(INTEL, 0x8c8e), board_ahci }, /* 9 Series RAID */ - { PCI_VDEVICE(INTEL, 0x8c8f), board_ahci_low_power }, /* 9 Series M RAID */ - { PCI_VDEVICE(INTEL, 0x9d03), board_ahci_low_power }, /* Sunrise LP AHCI */ - { PCI_VDEVICE(INTEL, 0x9d05), board_ahci_low_power }, /* Sunrise LP RAID */ - { PCI_VDEVICE(INTEL, 0x9d07), board_ahci_low_power }, /* Sunrise LP RAID */ - { PCI_VDEVICE(INTEL, 0xa102), board_ahci }, /* Sunrise Point-H AHCI */ - { PCI_VDEVICE(INTEL, 0xa103), board_ahci_low_power }, /* Sunrise M AHCI */ - { PCI_VDEVICE(INTEL, 0xa105), board_ahci }, /* Sunrise Point-H RAID */ - { PCI_VDEVICE(INTEL, 0xa106), board_ahci }, /* Sunrise Point-H RAID */ - { PCI_VDEVICE(INTEL, 0xa107), board_ahci_low_power }, /* Sunrise M RAID */ - { PCI_VDEVICE(INTEL, 0xa10f), board_ahci }, /* Sunrise Point-H RAID */ - { PCI_VDEVICE(INTEL, 0xa182), board_ahci }, /* Lewisburg AHCI*/ - { PCI_VDEVICE(INTEL, 0xa186), board_ahci }, /* Lewisburg RAID*/ - { PCI_VDEVICE(INTEL, 0xa1d2), board_ahci }, /* Lewisburg RAID*/ - { PCI_VDEVICE(INTEL, 0xa1d6), board_ahci }, /* Lewisburg RAID*/ - { PCI_VDEVICE(INTEL, 0xa202), board_ahci }, /* Lewisburg AHCI*/ - { PCI_VDEVICE(INTEL, 0xa206), board_ahci }, /* Lewisburg RAID*/ - { PCI_VDEVICE(INTEL, 0xa252), board_ahci }, /* Lewisburg RAID*/ - { PCI_VDEVICE(INTEL, 0xa256), board_ahci }, /* Lewisburg RAID*/ - { PCI_VDEVICE(INTEL, 0xa356), board_ahci }, /* Cannon Lake PCH-H RAID */ - { PCI_VDEVICE(INTEL, 0x06d7), board_ahci }, /* Comet Lake-H RAID */ - { PCI_VDEVICE(INTEL, 0xa386), board_ahci }, /* Comet Lake PCH-V RAID */ - { PCI_VDEVICE(INTEL, 0x0f22), board_ahci_low_power }, /* Bay Trail AHCI */ - { PCI_VDEVICE(INTEL, 0x0f23), board_ahci_low_power }, /* Bay Trail AHCI */ - { PCI_VDEVICE(INTEL, 0x22a3), board_ahci_low_power }, /* Cherry Tr. AHCI */ - { PCI_VDEVICE(INTEL, 0x5ae3), board_ahci_low_power }, /* ApolloLake AHCI */ - { PCI_VDEVICE(INTEL, 0x34d3), board_ahci_low_power }, /* Ice Lake LP AHCI */ - { PCI_VDEVICE(INTEL, 0x02d3), board_ahci_low_power }, /* Comet Lake PCH-U AHCI */ - { PCI_VDEVICE(INTEL, 0x02d7), board_ahci_low_power }, /* Comet Lake PCH RAID */ + { PCI_VDEVICE(INTEL, 0x2823), board_ahci_pcs_quirk }, /* Wellsburg/Lewisburg AHCI*/ + { PCI_VDEVICE(INTEL, 0x2826), board_ahci_pcs_quirk }, /* *burg SATA0 'RAID' */ + { PCI_VDEVICE(INTEL, 0x2827), board_ahci_pcs_quirk }, /* *burg SATA1 'RAID' */ + { PCI_VDEVICE(INTEL, 0x282f), board_ahci_pcs_quirk }, /* *burg SATA2 'RAID' */ + { PCI_VDEVICE(INTEL, 0x43d4), board_ahci_pcs_quirk }, /* Rocket Lake PCH-H RAID */ + { PCI_VDEVICE(INTEL, 0x43d5), board_ahci_pcs_quirk }, /* Rocket Lake PCH-H RAID */ + { PCI_VDEVICE(INTEL, 0x43d6), board_ahci_pcs_quirk }, /* Rocket Lake PCH-H RAID */ + { PCI_VDEVICE(INTEL, 0x43d7), board_ahci_pcs_quirk }, /* Rocket Lake PCH-H RAID */ + { PCI_VDEVICE(INTEL, 0x8d02), board_ahci_pcs_quirk }, /* Wellsburg AHCI */ + { PCI_VDEVICE(INTEL, 0x8d04), board_ahci_pcs_quirk }, /* Wellsburg RAID */ + { PCI_VDEVICE(INTEL, 0x8d06), board_ahci_pcs_quirk }, /* Wellsburg RAID */ + { PCI_VDEVICE(INTEL, 0x8d0e), board_ahci_pcs_quirk }, /* Wellsburg RAID */ + { PCI_VDEVICE(INTEL, 0x8d62), board_ahci_pcs_quirk }, /* Wellsburg AHCI */ + { PCI_VDEVICE(INTEL, 0x8d64), board_ahci_pcs_quirk }, /* Wellsburg RAID */ + { PCI_VDEVICE(INTEL, 0x8d66), board_ahci_pcs_quirk }, /* Wellsburg RAID */ + { PCI_VDEVICE(INTEL, 0x8d6e), board_ahci_pcs_quirk }, /* Wellsburg RAID */ + { PCI_VDEVICE(INTEL, 0x23a3), board_ahci_pcs_quirk }, /* Coleto Creek AHCI */ + { PCI_VDEVICE(INTEL, 0x9c83), board_ahci_pcs_quirk }, /* Wildcat LP AHCI */ + { PCI_VDEVICE(INTEL, 0x9c85), board_ahci_pcs_quirk }, /* Wildcat LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c87), board_ahci_pcs_quirk }, /* Wildcat LP RAID */ + { PCI_VDEVICE(INTEL, 0x9c8f), board_ahci_pcs_quirk }, /* Wildcat LP RAID */ + { PCI_VDEVICE(INTEL, 0x8c82), board_ahci_pcs_quirk }, /* 9 Series AHCI */ + { PCI_VDEVICE(INTEL, 0x8c83), board_ahci_pcs_quirk }, /* 9 Series M AHCI */ + { PCI_VDEVICE(INTEL, 0x8c84), board_ahci_pcs_quirk }, /* 9 Series RAID */ + { PCI_VDEVICE(INTEL, 0x8c85), board_ahci_pcs_quirk }, /* 9 Series M RAID */ + { PCI_VDEVICE(INTEL, 0x8c86), board_ahci_pcs_quirk }, /* 9 Series RAID */ + { PCI_VDEVICE(INTEL, 0x8c87), board_ahci_pcs_quirk }, /* 9 Series M RAID */ + { PCI_VDEVICE(INTEL, 0x8c8e), board_ahci_pcs_quirk }, /* 9 Series RAID */ + { PCI_VDEVICE(INTEL, 0x8c8f), board_ahci_pcs_quirk }, /* 9 Series M RAID */ + { PCI_VDEVICE(INTEL, 0x9d03), board_ahci_pcs_quirk }, /* Sunrise LP AHCI */ + { PCI_VDEVICE(INTEL, 0x9d05), board_ahci_pcs_quirk }, /* Sunrise LP RAID */ + { PCI_VDEVICE(INTEL, 0x9d07), board_ahci_pcs_quirk }, /* Sunrise LP RAID */ + { PCI_VDEVICE(INTEL, 0xa102), board_ahci_pcs_quirk }, /* Sunrise Point-H AHCI */ + { PCI_VDEVICE(INTEL, 0xa103), board_ahci_pcs_quirk }, /* Sunrise M AHCI */ + { PCI_VDEVICE(INTEL, 0xa105), board_ahci_pcs_quirk }, /* Sunrise Point-H RAID */ + { PCI_VDEVICE(INTEL, 0xa106), board_ahci_pcs_quirk }, /* Sunrise Point-H RAID */ + { PCI_VDEVICE(INTEL, 0xa107), board_ahci_pcs_quirk }, /* Sunrise M RAID */ + { PCI_VDEVICE(INTEL, 0xa10f), board_ahci_pcs_quirk }, /* Sunrise Point-H RAID */ + { PCI_VDEVICE(INTEL, 0xa182), board_ahci_pcs_quirk }, /* Lewisburg AHCI*/ + { PCI_VDEVICE(INTEL, 0xa186), board_ahci_pcs_quirk }, /* Lewisburg RAID*/ + { PCI_VDEVICE(INTEL, 0xa1d2), board_ahci_pcs_quirk }, /* Lewisburg RAID*/ + { PCI_VDEVICE(INTEL, 0xa1d6), board_ahci_pcs_quirk }, /* Lewisburg RAID*/ + { PCI_VDEVICE(INTEL, 0xa202), board_ahci_pcs_quirk }, /* Lewisburg AHCI*/ + { PCI_VDEVICE(INTEL, 0xa206), board_ahci_pcs_quirk }, /* Lewisburg RAID*/ + { PCI_VDEVICE(INTEL, 0xa252), board_ahci_pcs_quirk }, /* Lewisburg RAID*/ + { PCI_VDEVICE(INTEL, 0xa256), board_ahci_pcs_quirk }, /* Lewisburg RAID*/ + { PCI_VDEVICE(INTEL, 0xa356), board_ahci_pcs_quirk }, /* Cannon Lake PCH-H RAID */ + { PCI_VDEVICE(INTEL, 0x06d7), board_ahci_pcs_quirk }, /* Comet Lake-H RAID */ + { PCI_VDEVICE(INTEL, 0xa386), board_ahci_pcs_quirk }, /* Comet Lake PCH-V RAID */ + { PCI_VDEVICE(INTEL, 0x0f22), board_ahci_pcs_quirk }, /* Bay Trail AHCI */ + { PCI_VDEVICE(INTEL, 0x0f23), board_ahci_pcs_quirk_no_devslp }, /* Bay Trail AHCI */ + { PCI_VDEVICE(INTEL, 0x22a3), board_ahci_pcs_quirk }, /* Cherry Tr. AHCI */ + { PCI_VDEVICE(INTEL, 0x5ae3), board_ahci_pcs_quirk }, /* ApolloLake AHCI */ + { PCI_VDEVICE(INTEL, 0x34d3), board_ahci_pcs_quirk }, /* Ice Lake LP AHCI */ + { PCI_VDEVICE(INTEL, 0x02d3), board_ahci_pcs_quirk }, /* Comet Lake PCH-U AHCI */ + { PCI_VDEVICE(INTEL, 0x02d7), board_ahci_pcs_quirk }, /* Comet Lake PCH RAID */ /* Elkhart Lake IDs 0x4b60 & 0x4b62 https://sata-io.org/product/8803 not tested yet */ - { PCI_VDEVICE(INTEL, 0x4b63), board_ahci_low_power }, /* Elkhart Lake AHCI */ - { PCI_VDEVICE(INTEL, 0x7ae2), board_ahci_low_power }, /* Alder Lake-P AHCI */ + { PCI_VDEVICE(INTEL, 0x4b63), board_ahci_pcs_quirk }, /* Elkhart Lake AHCI */ + { PCI_VDEVICE(INTEL, 0x7ae2), board_ahci_pcs_quirk }, /* Alder Lake-P AHCI */ /* JMicron 360/1/3/5/6, match class to avoid IDE function */ { PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, @@ -459,14 +457,14 @@ static const struct pci_device_id ahci_pci_tbl[] = { { PCI_VDEVICE(AMD, 0x7800), board_ahci }, /* AMD Hudson-2 */ { PCI_VDEVICE(AMD, 0x7801), board_ahci_no_debounce_delay }, /* AMD Hudson-2 (AHCI mode) */ { PCI_VDEVICE(AMD, 0x7900), board_ahci }, /* AMD CZ */ - { PCI_VDEVICE(AMD, 0x7901), board_ahci_low_power }, /* AMD Green Sardine */ + { PCI_VDEVICE(AMD, 0x7901), board_ahci }, /* AMD Green Sardine */ /* AMD is using RAID class only for ahci controllers */ { PCI_VENDOR_ID_AMD, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_STORAGE_RAID << 8, 0xffffff, board_ahci }, /* Dell S140/S150 */ { PCI_VENDOR_ID_INTEL, PCI_ANY_ID, PCI_SUBVENDOR_ID_DELL, PCI_ANY_ID, - PCI_CLASS_STORAGE_RAID << 8, 0xffffff, board_ahci }, + PCI_CLASS_STORAGE_RAID << 8, 0xffffff, board_ahci_pcs_quirk }, /* VIA */ { PCI_VDEVICE(VIA, 0x3349), board_ahci_vt8251 }, /* VIA VT8251 */ @@ -623,8 +621,8 @@ static const struct pci_device_id ahci_pci_tbl[] = { * Samsung SSDs found on some macbooks. NCQ times out if MSI is * enabled. https://bugzilla.kernel.org/show_bug.cgi?id=60731 */ - { PCI_VDEVICE(SAMSUNG, 0x1600), board_ahci_nomsi }, - { PCI_VDEVICE(SAMSUNG, 0xa800), board_ahci_nomsi }, + { PCI_VDEVICE(SAMSUNG, 0x1600), board_ahci_no_msi }, + { PCI_VDEVICE(SAMSUNG, 0xa800), board_ahci_no_msi }, /* Enmotus */ { PCI_DEVICE(0x1c44, 0x8000), board_ahci }, @@ -1431,17 +1429,6 @@ static bool ahci_broken_online(struct pci_dev *pdev) return pdev->bus->number == (val >> 8) && pdev->devfn == (val & 0xff); } -static bool ahci_broken_devslp(struct pci_dev *pdev) -{ - /* device with broken DEVSLP but still showing SDS capability */ - static const struct pci_device_id ids[] = { - { PCI_VDEVICE(INTEL, 0x0f23)}, /* Valleyview SoC */ - {} - }; - - return pci_match_id(ids, pdev); -} - #ifdef CONFIG_ATA_ACPI static void ahci_gtf_filter_workaround(struct ata_host *host) { @@ -1650,14 +1637,31 @@ static int ahci_init_msi(struct pci_dev *pdev, unsigned int n_ports, return pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSIX); } -static void ahci_update_initial_lpm_policy(struct ata_port *ap, - struct ahci_host_priv *hpriv) +static void ahci_mark_external_port(struct ata_port *ap) { + struct ahci_host_priv *hpriv = ap->host->private_data; + void __iomem *port_mmio = ahci_port_base(ap); + u32 tmp; + + /* mark external ports (hotplug-capable, eSATA) */ + tmp = readl(port_mmio + PORT_CMD); + if (((tmp & PORT_CMD_ESP) && (hpriv->cap & HOST_CAP_SXS)) || + (tmp & PORT_CMD_HPCP)) + ap->pflags |= ATA_PFLAG_EXTERNAL; +} + +static void ahci_update_initial_lpm_policy(struct ata_port *ap) +{ + struct ahci_host_priv *hpriv = ap->host->private_data; int policy = CONFIG_SATA_MOBILE_LPM_POLICY; - - /* Ignore processing for chipsets that don't use policy */ - if (!(hpriv->flags & AHCI_HFLAG_USE_LPM_POLICY)) + /* + * AHCI contains a known incompatibility between LPM and hot-plug + * removal events, see 7.3.1 Hot Plug Removal Detection and Power + * Management Interaction in AHCI 1.3.1. Therefore, do not enable + * LPM if the port advertises itself as an external port. + */ + if (ap->pflags & ATA_PFLAG_EXTERNAL) return; /* user modified policy via module param */ @@ -1680,17 +1684,9 @@ update_policy: static void ahci_intel_pcs_quirk(struct pci_dev *pdev, struct ahci_host_priv *hpriv) { - const struct pci_device_id *id = pci_match_id(ahci_pci_tbl, pdev); u16 tmp16; - /* - * Only apply the 6-port PCS quirk for known legacy platforms. - */ - if (!id || id->vendor != PCI_VENDOR_ID_INTEL) - return; - - /* Skip applying the quirk on Denverton and beyond */ - if (((enum board_ids) id->driver_data) >= board_ahci_pcs7) + if (!(hpriv->flags & AHCI_HFLAG_INTEL_PCS_QUIRK)) return; /* @@ -1825,10 +1821,6 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) &dev_attr_remapped_nvme.attr, NULL); - /* must set flag prior to save config in order to take effect */ - if (ahci_broken_devslp(pdev)) - hpriv->flags |= AHCI_HFLAG_NO_DEVSLP; - #ifdef CONFIG_ARM64 if (pdev->vendor == PCI_VENDOR_ID_HUAWEI && pdev->device == 0xa235 && @@ -1942,7 +1934,9 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) if (ap->flags & ATA_FLAG_EM) ap->em_message_type = hpriv->em_msg_type; - ahci_update_initial_lpm_policy(ap, hpriv); + ahci_mark_external_port(ap); + + ahci_update_initial_lpm_policy(ap); /* disabled/not-implemented port */ if (!(hpriv->port_map & (1 << i))) diff --git a/drivers/ata/ahci.h b/drivers/ata/ahci.h index df8f8a1a3a34..344c87210d8f 100644 --- a/drivers/ata/ahci.h +++ b/drivers/ata/ahci.h @@ -241,13 +241,11 @@ enum { AHCI_HFLAG_YES_ALPM = BIT(23), /* force ALPM cap on */ AHCI_HFLAG_NO_WRITE_TO_RO = BIT(24), /* don't write to read only registers */ - AHCI_HFLAG_USE_LPM_POLICY = BIT(25), /* chipset that should use - SATA_MOBILE_LPM_POLICY - as default lpm_policy */ - AHCI_HFLAG_SUSPEND_PHYS = BIT(26), /* handle PHYs during + AHCI_HFLAG_SUSPEND_PHYS = BIT(25), /* handle PHYs during suspend/resume */ - AHCI_HFLAG_NO_SXS = BIT(28), /* SXS not supported */ - AHCI_HFLAG_43BIT_ONLY = BIT(29), /* 43bit DMA addr limit */ + AHCI_HFLAG_NO_SXS = BIT(26), /* SXS not supported */ + AHCI_HFLAG_43BIT_ONLY = BIT(27), /* 43bit DMA addr limit */ + AHCI_HFLAG_INTEL_PCS_QUIRK = BIT(28), /* apply Intel PCS quirk */ /* ap->flags bits */ diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c index 1a63200ea437..83431aae74d8 100644 --- a/drivers/ata/libahci.c +++ b/drivers/ata/libahci.c @@ -1280,10 +1280,8 @@ static void ahci_port_init(struct device *dev, struct ata_port *ap, int port_no, void __iomem *mmio, void __iomem *port_mmio) { - struct ahci_host_priv *hpriv = ap->host->private_data; const char *emsg = NULL; int rc; - u32 tmp; /* make sure port is not active */ rc = ahci_deinit_port(ap, &emsg); @@ -1291,11 +1289,6 @@ static void ahci_port_init(struct device *dev, struct ata_port *ap, dev_warn(dev, "%s (%d)\n", emsg, rc); ahci_port_clear_pending_irq(ap); - - /* mark esata ports */ - tmp = readl(port_mmio + PORT_CMD); - if ((tmp & PORT_CMD_ESP) && (hpriv->cap & HOST_CAP_SXS)) - ap->pflags |= ATA_PFLAG_EXTERNAL; } void ahci_init_controller(struct ata_host *host) @@ -2627,8 +2620,8 @@ void ahci_print_info(struct ata_host *host, const char *scc_s) speed_s = "?"; dev_info(host->dev, - "AHCI %02x%02x.%02x%02x " - "%u slots %u ports %s Gbps 0x%x impl %s mode\n" + "AHCI vers %02x%02x.%02x%02x, " + "%u command slots, %s Gbps, %s mode\n" , (vers >> 24) & 0xff, @@ -2637,11 +2630,17 @@ void ahci_print_info(struct ata_host *host, const char *scc_s) vers & 0xff, ((cap >> 8) & 0x1f) + 1, - (cap & 0x1f) + 1, speed_s, - impl, scc_s); + dev_info(host->dev, + "%u/%u ports implemented (port mask 0x%x)\n" + , + + hweight32(impl), + (cap & 0x1f) + 1, + impl); + dev_info(host->dev, "flags: " "%s%s%s%s%s%s%s" diff --git a/drivers/ata/pata_parport/pata_parport.c b/drivers/ata/pata_parport/pata_parport.c index a7adfdcb5e27..9a2cb9ca9d1d 100644 --- a/drivers/ata/pata_parport/pata_parport.c +++ b/drivers/ata/pata_parport/pata_parport.c @@ -464,7 +464,7 @@ static void pata_parport_bus_release(struct device *dev) /* nothing to do here but required to avoid warning on device removal */ } -static struct bus_type pata_parport_bus_type = { +static const struct bus_type pata_parport_bus_type = { .name = DRV_NAME, }; diff --git a/drivers/base/power/common.c b/drivers/base/power/common.c index 44ec20918a4d..327d168dd37a 100644 --- a/drivers/base/power/common.c +++ b/drivers/base/power/common.c @@ -167,6 +167,115 @@ struct device *dev_pm_domain_attach_by_name(struct device *dev, } EXPORT_SYMBOL_GPL(dev_pm_domain_attach_by_name); +/** + * dev_pm_domain_attach_list - Associate a device with its PM domains. + * @dev: The device used to lookup the PM domains for. + * @data: The data used for attaching to the PM domains. + * @list: An out-parameter with an allocated list of attached PM domains. + * + * This function helps to attach a device to its multiple PM domains. The + * caller, which is typically a driver's probe function, may provide a list of + * names for the PM domains that we should try to attach the device to, but it + * may also provide an empty list, in case the attach should be done for all of + * the available PM domains. + * + * Callers must ensure proper synchronization of this function with power + * management callbacks. + * + * Returns the number of attached PM domains or a negative error code in case of + * a failure. Note that, to detach the list of PM domains, the driver shall call + * dev_pm_domain_detach_list(), typically during the remove phase. + */ +int dev_pm_domain_attach_list(struct device *dev, + const struct dev_pm_domain_attach_data *data, + struct dev_pm_domain_list **list) +{ + struct device_node *np = dev->of_node; + struct dev_pm_domain_list *pds; + struct device *pd_dev = NULL; + int ret, i, num_pds = 0; + bool by_id = true; + u32 pd_flags = data ? data->pd_flags : 0; + u32 link_flags = pd_flags & PD_FLAG_NO_DEV_LINK ? 0 : + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME; + + if (dev->pm_domain) + return -EEXIST; + + /* For now this is limited to OF based platforms. */ + if (!np) + return 0; + + if (data && data->pd_names) { + num_pds = data->num_pd_names; + by_id = false; + } else { + num_pds = of_count_phandle_with_args(np, "power-domains", + "#power-domain-cells"); + } + + if (num_pds <= 0) + return 0; + + pds = devm_kzalloc(dev, sizeof(*pds), GFP_KERNEL); + if (!pds) + return -ENOMEM; + + pds->pd_devs = devm_kcalloc(dev, num_pds, sizeof(*pds->pd_devs), + GFP_KERNEL); + if (!pds->pd_devs) + return -ENOMEM; + + pds->pd_links = devm_kcalloc(dev, num_pds, sizeof(*pds->pd_links), + GFP_KERNEL); + if (!pds->pd_links) + return -ENOMEM; + + if (link_flags && pd_flags & PD_FLAG_DEV_LINK_ON) + link_flags |= DL_FLAG_RPM_ACTIVE; + + for (i = 0; i < num_pds; i++) { + if (by_id) + pd_dev = dev_pm_domain_attach_by_id(dev, i); + else + pd_dev = dev_pm_domain_attach_by_name(dev, + data->pd_names[i]); + if (IS_ERR_OR_NULL(pd_dev)) { + ret = pd_dev ? PTR_ERR(pd_dev) : -ENODEV; + goto err_attach; + } + + if (link_flags) { + struct device_link *link; + + link = device_link_add(dev, pd_dev, link_flags); + if (!link) { + ret = -ENODEV; + goto err_link; + } + + pds->pd_links[i] = link; + } + + pds->pd_devs[i] = pd_dev; + } + + pds->num_pds = num_pds; + *list = pds; + return num_pds; + +err_link: + dev_pm_domain_detach(pd_dev, true); +err_attach: + while (--i >= 0) { + if (pds->pd_links[i]) + device_link_del(pds->pd_links[i]); + dev_pm_domain_detach(pds->pd_devs[i], true); + } + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_domain_attach_list); + /** * dev_pm_domain_detach - Detach a device from its PM domain. * @dev: Device to detach. @@ -187,6 +296,31 @@ void dev_pm_domain_detach(struct device *dev, bool power_off) } EXPORT_SYMBOL_GPL(dev_pm_domain_detach); +/** + * dev_pm_domain_detach_list - Detach a list of PM domains. + * @list: The list of PM domains to detach. + * + * This function reverse the actions from dev_pm_domain_attach_list(). + * Typically it should be invoked during the remove phase from drivers. + * + * Callers must ensure proper synchronization of this function with power + * management callbacks. + */ +void dev_pm_domain_detach_list(struct dev_pm_domain_list *list) +{ + int i; + + if (!list) + return; + + for (i = 0; i < list->num_pds; i++) { + if (list->pd_links[i]) + device_link_del(list->pd_links[i]); + dev_pm_domain_detach(list->pd_devs[i], true); + } +} +EXPORT_SYMBOL_GPL(dev_pm_domain_detach_list); + /** * dev_pm_domain_start - Start the device through its PM domain. * @dev: Device to start. diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index c69edf145f6e..2e8847bc3d70 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -61,7 +61,6 @@ static LIST_HEAD(dpm_suspended_list); static LIST_HEAD(dpm_late_early_list); static LIST_HEAD(dpm_noirq_list); -struct suspend_stats suspend_stats; static DEFINE_MUTEX(dpm_list_mtx); static pm_message_t pm_transition; @@ -579,6 +578,35 @@ bool dev_pm_skip_resume(struct device *dev) return !dev->power.must_resume; } +static bool is_async(struct device *dev) +{ + return dev->power.async_suspend && pm_async_enabled + && !pm_trace_is_enabled(); +} + +static bool dpm_async_fn(struct device *dev, async_func_t func) +{ + reinit_completion(&dev->power.completion); + + if (is_async(dev)) { + dev->power.async_in_progress = true; + + get_device(dev); + + if (async_schedule_dev_nocall(func, dev)) + return true; + + put_device(dev); + } + /* + * Because async_schedule_dev_nocall() above has returned false or it + * has not been called at all, func() is not running and it is safe to + * update the async_in_progress flag without extra synchronization. + */ + dev->power.async_in_progress = false; + return false; +} + /** * device_resume_noirq - Execute a "noirq resume" callback for given device. * @dev: Device to handle. @@ -658,42 +686,12 @@ Out: TRACE_RESUME(error); if (error) { - suspend_stats.failed_resume_noirq++; - dpm_save_failed_step(SUSPEND_RESUME_NOIRQ); + async_error = error; dpm_save_failed_dev(dev_name(dev)); pm_dev_err(dev, state, async ? " async noirq" : " noirq", error); } } -static bool is_async(struct device *dev) -{ - return dev->power.async_suspend && pm_async_enabled - && !pm_trace_is_enabled(); -} - -static bool dpm_async_fn(struct device *dev, async_func_t func) -{ - reinit_completion(&dev->power.completion); - - if (is_async(dev)) { - dev->power.async_in_progress = true; - - get_device(dev); - - if (async_schedule_dev_nocall(func, dev)) - return true; - - put_device(dev); - } - /* - * Because async_schedule_dev_nocall() above has returned false or it - * has not been called at all, func() is not running and it is safe to - * update the async_in_progress flag without extra synchronization. - */ - dev->power.async_in_progress = false; - return false; -} - static void async_resume_noirq(void *data, async_cookie_t cookie) { struct device *dev = data; @@ -708,9 +706,12 @@ static void dpm_noirq_resume_devices(pm_message_t state) ktime_t starttime = ktime_get(); trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, true); - mutex_lock(&dpm_list_mtx); + + async_error = 0; pm_transition = state; + mutex_lock(&dpm_list_mtx); + /* * Trigger the resume of "async" devices upfront so they don't have to * wait for the "non-async" ones they don't depend on. @@ -737,6 +738,9 @@ static void dpm_noirq_resume_devices(pm_message_t state) mutex_unlock(&dpm_list_mtx); async_synchronize_full(); dpm_show_time(starttime, state, 0, "noirq"); + if (async_error) + dpm_save_failed_step(SUSPEND_RESUME_NOIRQ); + trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, false); } @@ -818,8 +822,7 @@ Out: complete_all(&dev->power.completion); if (error) { - suspend_stats.failed_resume_early++; - dpm_save_failed_step(SUSPEND_RESUME_EARLY); + async_error = error; dpm_save_failed_dev(dev_name(dev)); pm_dev_err(dev, state, async ? " async early" : " early", error); } @@ -843,9 +846,12 @@ void dpm_resume_early(pm_message_t state) ktime_t starttime = ktime_get(); trace_suspend_resume(TPS("dpm_resume_early"), state.event, true); - mutex_lock(&dpm_list_mtx); + + async_error = 0; pm_transition = state; + mutex_lock(&dpm_list_mtx); + /* * Trigger the resume of "async" devices upfront so they don't have to * wait for the "non-async" ones they don't depend on. @@ -872,6 +878,9 @@ void dpm_resume_early(pm_message_t state) mutex_unlock(&dpm_list_mtx); async_synchronize_full(); dpm_show_time(starttime, state, 0, "early"); + if (async_error) + dpm_save_failed_step(SUSPEND_RESUME_EARLY); + trace_suspend_resume(TPS("dpm_resume_early"), state.event, false); } @@ -975,8 +984,7 @@ static void device_resume(struct device *dev, pm_message_t state, bool async) TRACE_RESUME(error); if (error) { - suspend_stats.failed_resume++; - dpm_save_failed_step(SUSPEND_RESUME); + async_error = error; dpm_save_failed_dev(dev_name(dev)); pm_dev_err(dev, state, async ? " async" : "", error); } @@ -1005,10 +1013,11 @@ void dpm_resume(pm_message_t state) trace_suspend_resume(TPS("dpm_resume"), state.event, true); might_sleep(); - mutex_lock(&dpm_list_mtx); pm_transition = state; async_error = 0; + mutex_lock(&dpm_list_mtx); + /* * Trigger the resume of "async" devices upfront so they don't have to * wait for the "non-async" ones they don't depend on. @@ -1018,29 +1027,25 @@ void dpm_resume(pm_message_t state) while (!list_empty(&dpm_suspended_list)) { dev = to_device(dpm_suspended_list.next); - - get_device(dev); + list_move_tail(&dev->power.entry, &dpm_prepared_list); if (!dev->power.async_in_progress) { + get_device(dev); + mutex_unlock(&dpm_list_mtx); device_resume(dev, state, false); + put_device(dev); + mutex_lock(&dpm_list_mtx); } - - if (!list_empty(&dev->power.entry)) - list_move_tail(&dev->power.entry, &dpm_prepared_list); - - mutex_unlock(&dpm_list_mtx); - - put_device(dev); - - mutex_lock(&dpm_list_mtx); } mutex_unlock(&dpm_list_mtx); async_synchronize_full(); dpm_show_time(starttime, state, 0, NULL); + if (async_error) + dpm_save_failed_step(SUSPEND_RESUME); cpufreq_resume(); devfreq_resume(); @@ -1188,7 +1193,7 @@ static void dpm_superior_set_must_resume(struct device *dev) } /** - * __device_suspend_noirq - Execute a "noirq suspend" callback for given device. + * device_suspend_noirq - Execute a "noirq suspend" callback for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. * @async: If true, the device is being suspended asynchronously. @@ -1196,7 +1201,7 @@ static void dpm_superior_set_must_resume(struct device *dev) * The driver of @dev will not receive interrupts while this function is being * executed. */ -static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool async) +static int device_suspend_noirq(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; const char *info = NULL; @@ -1243,6 +1248,8 @@ Run: async_error = error; log_suspend_abort_reason("Device %s failed to %s noirq: error %d", dev_name(dev), pm_verb(state.event), error); + dpm_save_failed_dev(dev_name(dev)); + pm_dev_err(dev, state, async ? " async noirq" : " noirq", error); goto Complete; } @@ -1272,54 +1279,37 @@ Complete: static void async_suspend_noirq(void *data, async_cookie_t cookie) { struct device *dev = data; - int error; - - error = __device_suspend_noirq(dev, pm_transition, true); - if (error) { - dpm_save_failed_dev(dev_name(dev)); - pm_dev_err(dev, pm_transition, " async", error); - } + device_suspend_noirq(dev, pm_transition, true); put_device(dev); } -static int device_suspend_noirq(struct device *dev) -{ - if (dpm_async_fn(dev, async_suspend_noirq)) - return 0; - - return __device_suspend_noirq(dev, pm_transition, false); -} - static int dpm_noirq_suspend_devices(pm_message_t state) { ktime_t starttime = ktime_get(); int error = 0; trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, true); - mutex_lock(&dpm_list_mtx); + pm_transition = state; async_error = 0; + mutex_lock(&dpm_list_mtx); + while (!list_empty(&dpm_late_early_list)) { struct device *dev = to_device(dpm_late_early_list.prev); + list_move(&dev->power.entry, &dpm_noirq_list); + + if (dpm_async_fn(dev, async_suspend_noirq)) + continue; + get_device(dev); - mutex_unlock(&dpm_list_mtx); - - error = device_suspend_noirq(dev); - - mutex_lock(&dpm_list_mtx); - - if (error) { - pm_dev_err(dev, state, " noirq", error); - dpm_save_failed_dev(dev_name(dev)); - } else if (!list_empty(&dev->power.entry)) { - list_move(&dev->power.entry, &dpm_noirq_list); - } mutex_unlock(&dpm_list_mtx); + error = device_suspend_noirq(dev, state, false); + put_device(dev); mutex_lock(&dpm_list_mtx); @@ -1327,15 +1317,16 @@ static int dpm_noirq_suspend_devices(pm_message_t state) if (error || async_error) break; } + mutex_unlock(&dpm_list_mtx); + async_synchronize_full(); if (!error) error = async_error; - if (error) { - suspend_stats.failed_suspend_noirq++; + if (error) dpm_save_failed_step(SUSPEND_SUSPEND_NOIRQ); - } + dpm_show_time(starttime, state, error, "noirq"); trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, false); return error; @@ -1378,14 +1369,14 @@ static void dpm_propagate_wakeup_to_parent(struct device *dev) } /** - * __device_suspend_late - Execute a "late suspend" callback for given device. + * device_suspend_late - Execute a "late suspend" callback for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. * @async: If true, the device is being suspended asynchronously. * * Runtime PM is disabled for @dev while this function is being executed. */ -static int __device_suspend_late(struct device *dev, pm_message_t state, bool async) +static int device_suspend_late(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; const char *info = NULL; @@ -1439,6 +1430,8 @@ Run: async_error = error; log_suspend_abort_reason("Device %s failed to %s late: error %d", dev_name(dev), pm_verb(state.event), error); + dpm_save_failed_dev(dev_name(dev)); + pm_dev_err(dev, state, async ? " async late" : " late", error); goto Complete; } dpm_propagate_wakeup_to_parent(dev); @@ -1455,24 +1448,11 @@ Complete: static void async_suspend_late(void *data, async_cookie_t cookie) { struct device *dev = data; - int error; - error = __device_suspend_late(dev, pm_transition, true); - if (error) { - dpm_save_failed_dev(dev_name(dev)); - pm_dev_err(dev, pm_transition, " async", error); - } + device_suspend_late(dev, pm_transition, true); put_device(dev); } -static int device_suspend_late(struct device *dev) -{ - if (dpm_async_fn(dev, async_suspend_late)) - return 0; - - return __device_suspend_late(dev, pm_transition, false); -} - /** * dpm_suspend_late - Execute "late suspend" callbacks for all devices. * @state: PM transition of the system being carried out. @@ -1483,31 +1463,27 @@ int dpm_suspend_late(pm_message_t state) int error = 0; trace_suspend_resume(TPS("dpm_suspend_late"), state.event, true); - wake_up_all_idle_cpus(); - mutex_lock(&dpm_list_mtx); + pm_transition = state; async_error = 0; + wake_up_all_idle_cpus(); + + mutex_lock(&dpm_list_mtx); + while (!list_empty(&dpm_suspended_list)) { struct device *dev = to_device(dpm_suspended_list.prev); + list_move(&dev->power.entry, &dpm_late_early_list); + + if (dpm_async_fn(dev, async_suspend_late)) + continue; + get_device(dev); mutex_unlock(&dpm_list_mtx); - error = device_suspend_late(dev); - - mutex_lock(&dpm_list_mtx); - - if (!list_empty(&dev->power.entry)) - list_move(&dev->power.entry, &dpm_late_early_list); - - if (error) { - pm_dev_err(dev, state, " late", error); - dpm_save_failed_dev(dev_name(dev)); - } - - mutex_unlock(&dpm_list_mtx); + error = device_suspend_late(dev, state, false); put_device(dev); @@ -1516,12 +1492,14 @@ int dpm_suspend_late(pm_message_t state) if (error || async_error) break; } + mutex_unlock(&dpm_list_mtx); + async_synchronize_full(); if (!error) error = async_error; + if (error) { - suspend_stats.failed_suspend_late++; dpm_save_failed_step(SUSPEND_SUSPEND_LATE); dpm_resume_early(resume_event(state)); } @@ -1602,12 +1580,12 @@ static void dpm_clear_superiors_direct_complete(struct device *dev) } /** - * __device_suspend - Execute "suspend" callbacks for given device. + * device_suspend - Execute "suspend" callbacks for given device. * @dev: Device to handle. * @state: PM transition of the system being carried out. * @async: If true, the device is being suspended asynchronously. */ -static int __device_suspend(struct device *dev, pm_message_t state, bool async) +static int device_suspend(struct device *dev, pm_message_t state, bool async) { pm_callback_t callback = NULL; const char *info = NULL; @@ -1724,8 +1702,11 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async) dpm_watchdog_clear(&wd); Complete: - if (error) + if (error) { async_error = error; + dpm_save_failed_dev(dev_name(dev)); + pm_dev_err(dev, state, async ? " async" : "", error); + } complete_all(&dev->power.completion); TRACE_SUSPEND(error); @@ -1735,25 +1716,11 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async) static void async_suspend(void *data, async_cookie_t cookie) { struct device *dev = data; - int error; - - error = __device_suspend(dev, pm_transition, true); - if (error) { - dpm_save_failed_dev(dev_name(dev)); - pm_dev_err(dev, pm_transition, " async", error); - } + device_suspend(dev, pm_transition, true); put_device(dev); } -static int device_suspend(struct device *dev) -{ - if (dpm_async_fn(dev, async_suspend)) - return 0; - - return __device_suspend(dev, pm_transition, false); -} - /** * dpm_suspend - Execute "suspend" callbacks for all non-sysdev devices. * @state: PM transition of the system being carried out. @@ -1769,28 +1736,24 @@ int dpm_suspend(pm_message_t state) devfreq_suspend(); cpufreq_suspend(); - mutex_lock(&dpm_list_mtx); pm_transition = state; async_error = 0; + + mutex_lock(&dpm_list_mtx); + while (!list_empty(&dpm_prepared_list)) { struct device *dev = to_device(dpm_prepared_list.prev); + list_move(&dev->power.entry, &dpm_suspended_list); + + if (dpm_async_fn(dev, async_suspend)) + continue; + get_device(dev); mutex_unlock(&dpm_list_mtx); - error = device_suspend(dev); - - mutex_lock(&dpm_list_mtx); - - if (error) { - pm_dev_err(dev, state, "", error); - dpm_save_failed_dev(dev_name(dev)); - } else if (!list_empty(&dev->power.entry)) { - list_move(&dev->power.entry, &dpm_suspended_list); - } - - mutex_unlock(&dpm_list_mtx); + error = device_suspend(dev, state, false); put_device(dev); @@ -1799,14 +1762,16 @@ int dpm_suspend(pm_message_t state) if (error || async_error) break; } + mutex_unlock(&dpm_list_mtx); + async_synchronize_full(); if (!error) error = async_error; - if (error) { - suspend_stats.failed_suspend++; + + if (error) dpm_save_failed_step(SUSPEND_SUSPEND); - } + dpm_show_time(starttime, state, error, NULL); trace_suspend_resume(TPS("dpm_suspend"), state.event, false); return error; @@ -1960,11 +1925,11 @@ int dpm_suspend_start(pm_message_t state) int error; error = dpm_prepare(state); - if (error) { - suspend_stats.failed_prepare++; + if (error) dpm_save_failed_step(SUSPEND_PREPARE); - } else + else error = dpm_suspend(state); + dpm_show_time(starttime, state, error, "start"); return error; } diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index 05793c9fbb84..2ee45841486b 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -94,6 +94,7 @@ static void update_pm_runtime_accounting(struct device *dev) static void __update_runtime_status(struct device *dev, enum rpm_status status) { update_pm_runtime_accounting(dev); + trace_rpm_status(dev, status); dev->power.runtime_status = status; } @@ -1176,7 +1177,7 @@ int __pm_runtime_resume(struct device *dev, int rpmflags) EXPORT_SYMBOL_GPL(__pm_runtime_resume); /** - * pm_runtime_get_if_active - Conditionally bump up device usage counter. + * pm_runtime_get_conditional - Conditionally bump up device usage counter. * @dev: Device to handle. * @ign_usage_count: Whether or not to look at the current usage counter value. * @@ -1197,7 +1198,7 @@ EXPORT_SYMBOL_GPL(__pm_runtime_resume); * The caller is responsible for decrementing the runtime PM usage counter of * @dev after this function has returned a positive value for it. */ -int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count) +static int pm_runtime_get_conditional(struct device *dev, bool ign_usage_count) { unsigned long flags; int retval; @@ -1218,8 +1219,39 @@ int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count) return retval; } + +/** + * pm_runtime_get_if_active - Bump up runtime PM usage counter if the device is + * in active state + * @dev: Target device. + * + * Increment the runtime PM usage counter of @dev if its runtime PM status is + * %RPM_ACTIVE, in which case it returns 1. If the device is in a different + * state, 0 is returned. -EINVAL is returned if runtime PM is disabled for the + * device, in which case also the usage_count will remain unmodified. + */ +int pm_runtime_get_if_active(struct device *dev) +{ + return pm_runtime_get_conditional(dev, true); +} EXPORT_SYMBOL_GPL(pm_runtime_get_if_active); +/** + * pm_runtime_get_if_in_use - Conditionally bump up runtime PM usage counter. + * @dev: Target device. + * + * Increment the runtime PM usage counter of @dev if its runtime PM status is + * %RPM_ACTIVE and its runtime PM usage counter is greater than 0, in which case + * it returns 1. If the device is in a different state or its usage_count is 0, + * 0 is returned. -EINVAL is returned if runtime PM is disabled for the device, + * in which case also the usage_count will remain unmodified. + */ +int pm_runtime_get_if_in_use(struct device *dev) +{ + return pm_runtime_get_conditional(dev, false); +} +EXPORT_SYMBOL_GPL(pm_runtime_get_if_in_use); + /** * __pm_runtime_set_status - Set runtime PM status of a device. * @dev: Device to handle. diff --git a/drivers/base/power/wakeirq.c b/drivers/base/power/wakeirq.c index 42171f766dcb..5a5a9e978e85 100644 --- a/drivers/base/power/wakeirq.c +++ b/drivers/base/power/wakeirq.c @@ -313,8 +313,10 @@ void dev_pm_enable_wake_irq_complete(struct device *dev) return; if (wirq->status & WAKE_IRQ_DEDICATED_MANAGED && - wirq->status & WAKE_IRQ_DEDICATED_REVERSE) + wirq->status & WAKE_IRQ_DEDICATED_REVERSE) { enable_irq(wirq->irq); + wirq->status |= WAKE_IRQ_DEDICATED_ENABLED; + } } /** diff --git a/drivers/base/regmap/internal.h b/drivers/base/regmap/internal.h index 583dd5d7d46b..bcdb25bec77c 100644 --- a/drivers/base/regmap/internal.h +++ b/drivers/base/regmap/internal.h @@ -93,6 +93,7 @@ struct regmap { #endif unsigned int max_register; + bool max_register_is_set; bool (*writeable_reg)(struct device *dev, unsigned int reg); bool (*readable_reg)(struct device *dev, unsigned int reg); bool (*volatile_reg)(struct device *dev, unsigned int reg); diff --git a/drivers/base/regmap/regcache-flat.c b/drivers/base/regmap/regcache-flat.c index b7e4b2464102..9b17c77dec9d 100644 --- a/drivers/base/regmap/regcache-flat.c +++ b/drivers/base/regmap/regcache-flat.c @@ -23,7 +23,7 @@ static int regcache_flat_init(struct regmap *map) int i; unsigned int *cache; - if (!map || map->reg_stride_order < 0 || !map->max_register) + if (!map || map->reg_stride_order < 0 || !map->max_register_is_set) return -EINVAL; map->cache = kcalloc(regcache_flat_get_index(map, map->max_register) diff --git a/drivers/base/regmap/regcache.c b/drivers/base/regmap/regcache.c index ac63a73ccdaa..2e41cb12b8e2 100644 --- a/drivers/base/regmap/regcache.c +++ b/drivers/base/regmap/regcache.c @@ -187,8 +187,10 @@ int regcache_init(struct regmap *map, const struct regmap_config *config) return 0; } - if (!map->max_register && map->num_reg_defaults_raw) + if (!map->max_register_is_set && map->num_reg_defaults_raw) { map->max_register = (map->num_reg_defaults_raw - 1) * map->reg_stride; + map->max_register_is_set = true; + } if (map->cache_ops->init) { dev_dbg(map->dev, "Initializing %s cache\n", diff --git a/drivers/base/regmap/regmap-kunit.c b/drivers/base/regmap/regmap-kunit.c index 0d957c5f1bcc..bb2ab6129f38 100644 --- a/drivers/base/regmap/regmap-kunit.c +++ b/drivers/base/regmap/regmap-kunit.c @@ -1341,6 +1341,71 @@ static void raw_sync(struct kunit *test) regmap_exit(map); } +static void raw_ranges(struct kunit *test) +{ + struct raw_test_types *t = (struct raw_test_types *)test->param_value; + struct regmap *map; + struct regmap_config config; + struct regmap_ram_data *data; + unsigned int val; + int i; + + config = raw_regmap_config; + config.volatile_reg = test_range_all_volatile; + config.ranges = &test_range; + config.num_ranges = 1; + config.max_register = test_range.range_max; + + map = gen_raw_regmap(&config, t, &data); + KUNIT_ASSERT_FALSE(test, IS_ERR(map)); + if (IS_ERR(map)) + return; + + /* Reset the page to a non-zero value to trigger a change */ + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, test_range.selector_reg, + test_range.range_max)); + + /* Check we set the page and use the window for writes */ + data->written[test_range.selector_reg] = false; + data->written[test_range.window_start] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, test_range.range_min, 0)); + KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]); + KUNIT_EXPECT_TRUE(test, data->written[test_range.window_start]); + + data->written[test_range.selector_reg] = false; + data->written[test_range.window_start] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_write(map, + test_range.range_min + + test_range.window_len, + 0)); + KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]); + KUNIT_EXPECT_TRUE(test, data->written[test_range.window_start]); + + /* Same for reads */ + data->written[test_range.selector_reg] = false; + data->read[test_range.window_start] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, test_range.range_min, &val)); + KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]); + KUNIT_EXPECT_TRUE(test, data->read[test_range.window_start]); + + data->written[test_range.selector_reg] = false; + data->read[test_range.window_start] = false; + KUNIT_EXPECT_EQ(test, 0, regmap_read(map, + test_range.range_min + + test_range.window_len, + &val)); + KUNIT_EXPECT_TRUE(test, data->written[test_range.selector_reg]); + KUNIT_EXPECT_TRUE(test, data->read[test_range.window_start]); + + /* No physical access triggered in the virtual range */ + for (i = test_range.range_min; i < test_range.range_max; i++) { + KUNIT_EXPECT_FALSE(test, data->read[i]); + KUNIT_EXPECT_FALSE(test, data->written[i]); + } + + regmap_exit(map); +} + static struct kunit_case regmap_test_cases[] = { KUNIT_CASE_PARAM(basic_read_write, regcache_types_gen_params), KUNIT_CASE_PARAM(bulk_write, regcache_types_gen_params), @@ -1368,6 +1433,7 @@ static struct kunit_case regmap_test_cases[] = { KUNIT_CASE_PARAM(raw_write, raw_test_types_gen_params), KUNIT_CASE_PARAM(raw_noinc_write, raw_test_types_gen_params), KUNIT_CASE_PARAM(raw_sync, raw_test_cache_types_gen_params), + KUNIT_CASE_PARAM(raw_ranges, raw_test_cache_types_gen_params), {} }; diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c index 6db77d8e45f9..5cb425f6f02d 100644 --- a/drivers/base/regmap/regmap.c +++ b/drivers/base/regmap/regmap.c @@ -89,7 +89,7 @@ EXPORT_SYMBOL_GPL(regmap_check_range_table); bool regmap_writeable(struct regmap *map, unsigned int reg) { - if (map->max_register && reg > map->max_register) + if (map->max_register_is_set && reg > map->max_register) return false; if (map->writeable_reg) @@ -112,7 +112,7 @@ bool regmap_cached(struct regmap *map, unsigned int reg) if (!map->cache_ops) return false; - if (map->max_register && reg > map->max_register) + if (map->max_register_is_set && reg > map->max_register) return false; map->lock(map->lock_arg); @@ -129,7 +129,7 @@ bool regmap_readable(struct regmap *map, unsigned int reg) if (!map->reg_read) return false; - if (map->max_register && reg > map->max_register) + if (map->max_register_is_set && reg > map->max_register) return false; if (map->format.format_write) @@ -787,6 +787,7 @@ struct regmap *__regmap_init(struct device *dev, map->bus = bus; map->bus_context = bus_context; map->max_register = config->max_register; + map->max_register_is_set = map->max_register ?: config->max_register_is_0; map->wr_table = config->wr_table; map->rd_table = config->rd_table; map->volatile_table = config->volatile_table; @@ -1412,6 +1413,7 @@ int regmap_reinit_cache(struct regmap *map, const struct regmap_config *config) regmap_debugfs_exit(map); map->max_register = config->max_register; + map->max_register_is_set = map->max_register ?: config->max_register_is_0; map->writeable_reg = config->writeable_reg; map->readable_reg = config->readable_reg; map->volatile_reg = config->volatile_reg; @@ -3383,7 +3385,7 @@ EXPORT_SYMBOL_GPL(regmap_get_val_bytes); */ int regmap_get_max_register(struct regmap *map) { - return map->max_register ? map->max_register : -EINVAL; + return map->max_register_is_set ? map->max_register : -EINVAL; } EXPORT_SYMBOL_GPL(regmap_get_max_register); diff --git a/drivers/char/tpm/tpm_tis_spi_main.c b/drivers/char/tpm/tpm_tis_spi_main.c index c5c3197ee29f..c647de7b3709 100644 --- a/drivers/char/tpm/tpm_tis_spi_main.c +++ b/drivers/char/tpm/tpm_tis_spi_main.c @@ -146,7 +146,7 @@ static int tpm_tis_spi_transfer_full(struct tpm_tis_data *data, u32 addr, struct spi_transfer spi_xfer; u8 transfer_len; - spi_bus_lock(phy->spi_device->master); + spi_bus_lock(phy->spi_device->controller); while (len) { transfer_len = min_t(u16, len, MAX_SPI_FRAMESIZE); @@ -210,7 +210,7 @@ exit: spi_sync_locked(phy->spi_device, &m); } - spi_bus_unlock(phy->spi_device->master); + spi_bus_unlock(phy->spi_device->controller); return ret; } diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm index f911606897b8..a0ebad77666e 100644 --- a/drivers/cpufreq/Kconfig.arm +++ b/drivers/cpufreq/Kconfig.arm @@ -173,6 +173,7 @@ config ARM_QCOM_CPUFREQ_NVMEM config ARM_QCOM_CPUFREQ_HW tristate "QCOM CPUFreq HW driver" depends on ARCH_QCOM || COMPILE_TEST + depends on COMMON_CLK help Support for the CPUFreq HW driver. Some QCOM chipsets have a HW engine to offload the steps diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 1791d37fbc53..2015c9fcc3c9 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -49,6 +50,7 @@ #define AMD_PSTATE_TRANSITION_LATENCY 20000 #define AMD_PSTATE_TRANSITION_DELAY 1000 +#define AMD_PSTATE_PREFCORE_THRESHOLD 166 /* * TODO: We need more time to fine tune processors with shared memory solution @@ -64,6 +66,7 @@ static struct cpufreq_driver amd_pstate_driver; static struct cpufreq_driver amd_pstate_epp_driver; static int cppc_state = AMD_PSTATE_UNDEFINED; static bool cppc_enabled; +static bool amd_pstate_prefcore = true; /* * AMD Energy Preference Performance (EPP) @@ -297,13 +300,14 @@ static int pstate_init_perf(struct amd_cpudata *cpudata) if (ret) return ret; - /* - * TODO: Introduce AMD specific power feature. - * - * CPPC entry doesn't indicate the highest performance in some ASICs. + /* For platforms that do not support the preferred core feature, the + * highest_pef may be configured with 166 or 255, to avoid max frequency + * calculated wrongly. we take the AMD_CPPC_HIGHEST_PERF(cap1) value as + * the default max perf. */ - highest_perf = amd_get_highest_perf(); - if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1)) + if (cpudata->hw_prefcore) + highest_perf = AMD_PSTATE_PREFCORE_THRESHOLD; + else highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); WRITE_ONCE(cpudata->highest_perf, highest_perf); @@ -311,6 +315,7 @@ static int pstate_init_perf(struct amd_cpudata *cpudata) WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); + WRITE_ONCE(cpudata->prefcore_ranking, AMD_CPPC_HIGHEST_PERF(cap1)); WRITE_ONCE(cpudata->min_limit_perf, AMD_CPPC_LOWEST_PERF(cap1)); return 0; } @@ -324,8 +329,9 @@ static int cppc_init_perf(struct amd_cpudata *cpudata) if (ret) return ret; - highest_perf = amd_get_highest_perf(); - if (highest_perf > cppc_perf.highest_perf) + if (cpudata->hw_prefcore) + highest_perf = AMD_PSTATE_PREFCORE_THRESHOLD; + else highest_perf = cppc_perf.highest_perf; WRITE_ONCE(cpudata->highest_perf, highest_perf); @@ -334,6 +340,7 @@ static int cppc_init_perf(struct amd_cpudata *cpudata) WRITE_ONCE(cpudata->lowest_nonlinear_perf, cppc_perf.lowest_nonlinear_perf); WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); + WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); WRITE_ONCE(cpudata->min_limit_perf, cppc_perf.lowest_perf); if (cppc_state == AMD_PSTATE_ACTIVE) @@ -477,12 +484,19 @@ static int amd_pstate_verify(struct cpufreq_policy_data *policy) static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) { - u32 max_limit_perf, min_limit_perf; + u32 max_limit_perf, min_limit_perf, lowest_perf; struct amd_cpudata *cpudata = policy->driver_data; max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq); min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq); + lowest_perf = READ_ONCE(cpudata->lowest_perf); + if (min_limit_perf < lowest_perf) + min_limit_perf = lowest_perf; + + if (max_limit_perf < min_limit_perf) + max_limit_perf = min_limit_perf; + WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf); WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf); WRITE_ONCE(cpudata->max_limit_freq, policy->max); @@ -570,7 +584,7 @@ static void amd_pstate_adjust_perf(unsigned int cpu, if (target_perf < capacity) des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); - min_perf = READ_ONCE(cpudata->highest_perf); + min_perf = READ_ONCE(cpudata->lowest_perf); if (_min_perf < capacity) min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); @@ -706,6 +720,114 @@ static void amd_perf_ctl_reset(unsigned int cpu) wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); } +/* + * Set amd-pstate preferred core enable can't be done directly from cpufreq callbacks + * due to locking, so queue the work for later. + */ +static void amd_pstste_sched_prefcore_workfn(struct work_struct *work) +{ + sched_set_itmt_support(); +} +static DECLARE_WORK(sched_prefcore_work, amd_pstste_sched_prefcore_workfn); + +/* + * Get the highest performance register value. + * @cpu: CPU from which to get highest performance. + * @highest_perf: Return address. + * + * Return: 0 for success, -EIO otherwise. + */ +static int amd_pstate_get_highest_perf(int cpu, u32 *highest_perf) +{ + int ret; + + if (boot_cpu_has(X86_FEATURE_CPPC)) { + u64 cap1; + + ret = rdmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &cap1); + if (ret) + return ret; + WRITE_ONCE(*highest_perf, AMD_CPPC_HIGHEST_PERF(cap1)); + } else { + u64 cppc_highest_perf; + + ret = cppc_get_highest_perf(cpu, &cppc_highest_perf); + if (ret) + return ret; + WRITE_ONCE(*highest_perf, cppc_highest_perf); + } + + return (ret); +} + +#define CPPC_MAX_PERF U8_MAX + +static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) +{ + int ret, prio; + u32 highest_perf; + + ret = amd_pstate_get_highest_perf(cpudata->cpu, &highest_perf); + if (ret) + return; + + cpudata->hw_prefcore = true; + /* check if CPPC preferred core feature is enabled*/ + if (highest_perf < CPPC_MAX_PERF) + prio = (int)highest_perf; + else { + pr_debug("AMD CPPC preferred core is unsupported!\n"); + cpudata->hw_prefcore = false; + return; + } + + if (!amd_pstate_prefcore) + return; + + /* + * The priorities can be set regardless of whether or not + * sched_set_itmt_support(true) has been called and it is valid to + * update them at any time after it has been called. + */ + sched_set_itmt_core_prio(prio, cpudata->cpu); + + schedule_work(&sched_prefcore_work); +} + +static void amd_pstate_update_limits(unsigned int cpu) +{ + struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); + struct amd_cpudata *cpudata = policy->driver_data; + u32 prev_high = 0, cur_high = 0; + int ret; + bool highest_perf_changed = false; + + mutex_lock(&amd_pstate_driver_lock); + if ((!amd_pstate_prefcore) || (!cpudata->hw_prefcore)) + goto free_cpufreq_put; + + ret = amd_pstate_get_highest_perf(cpu, &cur_high); + if (ret) + goto free_cpufreq_put; + + prev_high = READ_ONCE(cpudata->prefcore_ranking); + if (prev_high != cur_high) { + highest_perf_changed = true; + WRITE_ONCE(cpudata->prefcore_ranking, cur_high); + + if (cur_high < CPPC_MAX_PERF) + sched_set_itmt_core_prio((int)cur_high, cpu); + } + +free_cpufreq_put: + cpufreq_cpu_put(policy); + + if (!highest_perf_changed) + cpufreq_update_policy(cpu); + + mutex_unlock(&amd_pstate_driver_lock); +} + static int amd_pstate_cpu_init(struct cpufreq_policy *policy) { int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; @@ -727,6 +849,8 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) cpudata->cpu = policy->cpu; + amd_pstate_init_prefcore(cpudata); + ret = amd_pstate_init_perf(cpudata); if (ret) goto free_cpudata1; @@ -877,6 +1001,28 @@ static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, return sysfs_emit(buf, "%u\n", perf); } +static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy, + char *buf) +{ + u32 perf; + struct amd_cpudata *cpudata = policy->driver_data; + + perf = READ_ONCE(cpudata->prefcore_ranking); + + return sysfs_emit(buf, "%u\n", perf); +} + +static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, + char *buf) +{ + bool hw_prefcore; + struct amd_cpudata *cpudata = policy->driver_data; + + hw_prefcore = READ_ONCE(cpudata->hw_prefcore); + + return sysfs_emit(buf, "%s\n", str_enabled_disabled(hw_prefcore)); +} + static ssize_t show_energy_performance_available_preferences( struct cpufreq_policy *policy, char *buf) { @@ -1074,18 +1220,29 @@ static ssize_t status_store(struct device *a, struct device_attribute *b, return ret < 0 ? ret : count; } +static ssize_t prefcore_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore)); +} + cpufreq_freq_attr_ro(amd_pstate_max_freq); cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); cpufreq_freq_attr_ro(amd_pstate_highest_perf); +cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); +cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); cpufreq_freq_attr_rw(energy_performance_preference); cpufreq_freq_attr_ro(energy_performance_available_preferences); static DEVICE_ATTR_RW(status); +static DEVICE_ATTR_RO(prefcore); static struct freq_attr *amd_pstate_attr[] = { &amd_pstate_max_freq, &amd_pstate_lowest_nonlinear_freq, &amd_pstate_highest_perf, + &amd_pstate_prefcore_ranking, + &amd_pstate_hw_prefcore, NULL, }; @@ -1093,6 +1250,8 @@ static struct freq_attr *amd_pstate_epp_attr[] = { &amd_pstate_max_freq, &amd_pstate_lowest_nonlinear_freq, &amd_pstate_highest_perf, + &amd_pstate_prefcore_ranking, + &amd_pstate_hw_prefcore, &energy_performance_preference, &energy_performance_available_preferences, NULL, @@ -1100,6 +1259,7 @@ static struct freq_attr *amd_pstate_epp_attr[] = { static struct attribute *pstate_global_attributes[] = { &dev_attr_status.attr, + &dev_attr_prefcore.attr, NULL }; @@ -1151,6 +1311,8 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) cpudata->cpu = policy->cpu; cpudata->epp_policy = 0; + amd_pstate_init_prefcore(cpudata); + ret = amd_pstate_init_perf(cpudata); if (ret) goto free_cpudata1; @@ -1232,6 +1394,12 @@ static void amd_pstate_epp_update_limit(struct cpufreq_policy *policy) max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq); min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq); + if (min_limit_perf < min_perf) + min_limit_perf = min_perf; + + if (max_limit_perf < min_limit_perf) + max_limit_perf = min_limit_perf; + WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf); WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf); @@ -1432,6 +1600,7 @@ static struct cpufreq_driver amd_pstate_driver = { .suspend = amd_pstate_cpu_suspend, .resume = amd_pstate_cpu_resume, .set_boost = amd_pstate_set_boost, + .update_limits = amd_pstate_update_limits, .name = "amd-pstate", .attr = amd_pstate_attr, }; @@ -1446,6 +1615,7 @@ static struct cpufreq_driver amd_pstate_epp_driver = { .online = amd_pstate_epp_cpu_online, .suspend = amd_pstate_epp_suspend, .resume = amd_pstate_epp_resume, + .update_limits = amd_pstate_update_limits, .name = "amd-pstate-epp", .attr = amd_pstate_epp_attr, }; @@ -1567,7 +1737,17 @@ static int __init amd_pstate_param(char *str) return amd_pstate_set_driver(mode_idx); } + +static int __init amd_prefcore_param(char *str) +{ + if (!strcmp(str, "disable")) + amd_pstate_prefcore = false; + + return 0; +} + early_param("amd_pstate", amd_pstate_param); +early_param("amd_prefcore", amd_prefcore_param); MODULE_AUTHOR("Huang Rui "); MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); diff --git a/drivers/cpufreq/brcmstb-avs-cpufreq.c b/drivers/cpufreq/brcmstb-avs-cpufreq.c index 35fb3a559ea9..1a1857b0a6f4 100644 --- a/drivers/cpufreq/brcmstb-avs-cpufreq.c +++ b/drivers/cpufreq/brcmstb-avs-cpufreq.c @@ -481,6 +481,8 @@ static bool brcm_avs_is_firmware_loaded(struct private_data *priv) static unsigned int brcm_avs_cpufreq_get(unsigned int cpu) { struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); + if (!policy) + return 0; struct private_data *priv = policy->driver_data; cpufreq_cpu_put(policy); diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c index bd1e1357cef8..b993a498084b 100644 --- a/drivers/cpufreq/cpufreq-dt-platdev.c +++ b/drivers/cpufreq/cpufreq-dt-platdev.c @@ -156,6 +156,7 @@ static const struct of_device_id blocklist[] __initconst = { { .compatible = "qcom,sc7280", }, { .compatible = "qcom,sc8180x", }, { .compatible = "qcom,sc8280xp", }, + { .compatible = "qcom,sdm670", }, { .compatible = "qcom,sdm845", }, { .compatible = "qcom,sdx75", }, { .compatible = "qcom,sm6115", }, diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 48b532edcd93..67cd7359c719 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -579,17 +579,26 @@ unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy) latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC; if (latency) { + unsigned int max_delay_us = 2 * MSEC_PER_SEC; + /* - * For platforms that can change the frequency very fast (< 10 + * If the platform already has high transition_latency, use it + * as-is. + */ + if (latency > max_delay_us) + return latency; + + /* + * For platforms that can change the frequency very fast (< 2 * us), the above formula gives a decent transition delay. But * for platforms where transition_latency is in milliseconds, it * ends up giving unrealistic values. * - * Cap the default transition delay to 10 ms, which seems to be + * Cap the default transition delay to 2 ms, which seems to be * a reasonable amount of time after which we should reevaluate * the frequency. */ - return min(latency * LATENCY_MULTIPLIER, (unsigned int)10000); + return min(latency * LATENCY_MULTIPLIER, max_delay_us); } return LATENCY_MULTIPLIER; @@ -1582,7 +1591,8 @@ static int cpufreq_online(unsigned int cpu) if (cpufreq_driver->ready) cpufreq_driver->ready(policy); - if (cpufreq_thermal_control_enabled(cpufreq_driver)) + /* Register cpufreq cooling only for a new policy */ + if (new_policy && cpufreq_thermal_control_enabled(cpufreq_driver)) policy->cdev = of_cpufreq_cooling_register(policy); pr_debug("initialization complete\n"); @@ -1666,11 +1676,6 @@ static void __cpufreq_offline(unsigned int cpu, struct cpufreq_policy *policy) else policy->last_policy = policy->policy; - if (cpufreq_thermal_control_enabled(cpufreq_driver)) { - cpufreq_cooling_unregister(policy->cdev); - policy->cdev = NULL; - } - if (has_target()) cpufreq_exit_governor(policy); @@ -1731,6 +1736,15 @@ static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif) return; } + /* + * Unregister cpufreq cooling once all the CPUs of the policy are + * removed. + */ + if (cpufreq_thermal_control_enabled(cpufreq_driver)) { + cpufreq_cooling_unregister(policy->cdev); + policy->cdev = NULL; + } + /* We did light-weight exit earlier, do full tear down now */ if (cpufreq_driver->offline) cpufreq_driver->exit(policy); diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c index c52d19d67557..a7c38b8b3e78 100644 --- a/drivers/cpufreq/cpufreq_ondemand.c +++ b/drivers/cpufreq/cpufreq_ondemand.c @@ -22,7 +22,6 @@ #define DEF_SAMPLING_DOWN_FACTOR (1) #define MAX_SAMPLING_DOWN_FACTOR (100000) #define MICRO_FREQUENCY_UP_THRESHOLD (95) -#define MICRO_FREQUENCY_MIN_SAMPLE_RATE (10000) #define MIN_FREQUENCY_UP_THRESHOLD (1) #define MAX_FREQUENCY_UP_THRESHOLD (100) diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c index 33728c242f66..c20d3ecc5a81 100644 --- a/drivers/cpufreq/imx6q-cpufreq.c +++ b/drivers/cpufreq/imx6q-cpufreq.c @@ -14,6 +14,8 @@ #include #include #include +#include +#include #define PU_SOC_VOLTAGE_NORMAL 1250000 #define PU_SOC_VOLTAGE_HIGH 1275000 @@ -225,8 +227,6 @@ static void imx6x_disable_freq_in_opp(struct device *dev, unsigned long freq) static int imx6q_opp_check_speed_grading(struct device *dev) { - struct device_node *np; - void __iomem *base; u32 val; int ret; @@ -235,16 +235,11 @@ static int imx6q_opp_check_speed_grading(struct device *dev) if (ret) return ret; } else { - np = of_find_compatible_node(NULL, NULL, "fsl,imx6q-ocotp"); - if (!np) - return -ENOENT; + struct regmap *ocotp; - base = of_iomap(np, 0); - of_node_put(np); - if (!base) { - dev_err(dev, "failed to map ocotp\n"); - return -EFAULT; - } + ocotp = syscon_regmap_lookup_by_compatible("fsl,imx6q-ocotp"); + if (IS_ERR(ocotp)) + return -ENOENT; /* * SPEED_GRADING[1:0] defines the max speed of ARM: @@ -254,8 +249,7 @@ static int imx6q_opp_check_speed_grading(struct device *dev) * 2b'00: 792000000Hz; * We need to set the max speed of ARM according to fuse map. */ - val = readl_relaxed(base + OCOTP_CFG3); - iounmap(base); + regmap_read(ocotp, OCOTP_CFG3, &val); } val >>= OCOTP_CFG3_SPEED_SHIFT; @@ -290,25 +284,16 @@ static int imx6ul_opp_check_speed_grading(struct device *dev) if (ret) return ret; } else { - struct device_node *np; - void __iomem *base; + struct regmap *ocotp; - np = of_find_compatible_node(NULL, NULL, "fsl,imx6ul-ocotp"); - if (!np) - np = of_find_compatible_node(NULL, NULL, - "fsl,imx6ull-ocotp"); - if (!np) + ocotp = syscon_regmap_lookup_by_compatible("fsl,imx6ul-ocotp"); + if (IS_ERR(ocotp)) + ocotp = syscon_regmap_lookup_by_compatible("fsl,imx6ull-ocotp"); + + if (IS_ERR(ocotp)) return -ENOENT; - base = of_iomap(np, 0); - of_node_put(np); - if (!base) { - dev_err(dev, "failed to map ocotp\n"); - return -EFAULT; - } - - val = readl_relaxed(base + OCOTP_CFG3); - iounmap(base); + regmap_read(ocotp, OCOTP_CFG3, &val); } /* diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 79619227ea51..dbbf299f4219 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -201,8 +202,6 @@ struct global_params { * @prev_aperf: Last APERF value read from APERF MSR * @prev_mperf: Last MPERF value read from MPERF MSR * @prev_tsc: Last timestamp counter (TSC) value - * @prev_cummulative_iowait: IO Wait time difference from last and - * current sample * @sample: Storage for storing last Sample data * @min_perf_ratio: Minimum capacity in terms of PERF or HWP ratios * @max_perf_ratio: Maximum capacity in terms of PERF or HWP ratios @@ -241,7 +240,6 @@ struct cpudata { u64 prev_aperf; u64 prev_mperf; u64 prev_tsc; - u64 prev_cummulative_iowait; struct sample sample; int32_t min_perf_ratio; int32_t max_perf_ratio; @@ -3407,14 +3405,31 @@ static bool intel_pstate_hwp_is_enabled(void) return !!(value & 0x1); } -static const struct x86_cpu_id intel_epp_balance_perf[] = { +#define POWERSAVE_MASK GENMASK(7, 0) +#define BALANCE_POWER_MASK GENMASK(15, 8) +#define BALANCE_PERFORMANCE_MASK GENMASK(23, 16) +#define PERFORMANCE_MASK GENMASK(31, 24) + +#define HWP_SET_EPP_VALUES(powersave, balance_power, balance_perf, performance) \ + (FIELD_PREP_CONST(POWERSAVE_MASK, powersave) |\ + FIELD_PREP_CONST(BALANCE_POWER_MASK, balance_power) |\ + FIELD_PREP_CONST(BALANCE_PERFORMANCE_MASK, balance_perf) |\ + FIELD_PREP_CONST(PERFORMANCE_MASK, performance)) + +#define HWP_SET_DEF_BALANCE_PERF_EPP(balance_perf) \ + (HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE, HWP_EPP_BALANCE_POWERSAVE,\ + balance_perf, HWP_EPP_PERFORMANCE)) + +static const struct x86_cpu_id intel_epp_default[] = { /* * Set EPP value as 102, this is the max suggested EPP * which can result in one core turbo frequency for * AlderLake Mobile CPUs. */ - X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, 102), - X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, 32), + X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, HWP_SET_DEF_BALANCE_PERF_EPP(102)), + X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, HWP_SET_DEF_BALANCE_PERF_EPP(32)), + X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE_L, HWP_SET_EPP_VALUES(HWP_EPP_POWERSAVE, + HWP_EPP_BALANCE_POWERSAVE, 115, 16)), {} }; @@ -3512,11 +3527,24 @@ hwp_cpu_matched: intel_pstate_sysfs_expose_params(); if (hwp_active) { - const struct x86_cpu_id *id = x86_match_cpu(intel_epp_balance_perf); + const struct x86_cpu_id *id = x86_match_cpu(intel_epp_default); const struct x86_cpu_id *hybrid_id = x86_match_cpu(intel_hybrid_scaling_factor); - if (id) - epp_values[EPP_INDEX_BALANCE_PERFORMANCE] = id->driver_data; + if (id) { + epp_values[EPP_INDEX_POWERSAVE] = + FIELD_GET(POWERSAVE_MASK, id->driver_data); + epp_values[EPP_INDEX_BALANCE_POWERSAVE] = + FIELD_GET(BALANCE_POWER_MASK, id->driver_data); + epp_values[EPP_INDEX_BALANCE_PERFORMANCE] = + FIELD_GET(BALANCE_PERFORMANCE_MASK, id->driver_data); + epp_values[EPP_INDEX_PERFORMANCE] = + FIELD_GET(PERFORMANCE_MASK, id->driver_data); + pr_debug("Updated EPPs powersave:%x balanced power:%x balanced perf:%x performance:%x\n", + epp_values[EPP_INDEX_POWERSAVE], + epp_values[EPP_INDEX_BALANCE_POWERSAVE], + epp_values[EPP_INDEX_BALANCE_PERFORMANCE], + epp_values[EPP_INDEX_PERFORMANCE]); + } if (hybrid_id) { hybrid_scaling_factor = hybrid_id->driver_data; diff --git a/drivers/cpufreq/mediatek-cpufreq-hw.c b/drivers/cpufreq/mediatek-cpufreq-hw.c index d46afb3c0092..8d097dcddda4 100644 --- a/drivers/cpufreq/mediatek-cpufreq-hw.c +++ b/drivers/cpufreq/mediatek-cpufreq-hw.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #define LUT_MAX_ENTRIES 32U @@ -300,7 +301,23 @@ static struct cpufreq_driver cpufreq_mtk_hw_driver = { static int mtk_cpufreq_hw_driver_probe(struct platform_device *pdev) { const void *data; - int ret; + int ret, cpu; + struct device *cpu_dev; + struct regulator *cpu_reg; + + /* Make sure that all CPU supplies are available before proceeding. */ + for_each_possible_cpu(cpu) { + cpu_dev = get_cpu_device(cpu); + if (!cpu_dev) + return dev_err_probe(&pdev->dev, -EPROBE_DEFER, + "Failed to get cpu%d device\n", cpu); + + cpu_reg = devm_regulator_get(cpu_dev, "cpu"); + if (IS_ERR(cpu_reg)) + return dev_err_probe(&pdev->dev, PTR_ERR(cpu_reg), + "CPU%d regulator get failed\n", cpu); + } + data = of_device_get_match_data(&pdev->dev); if (!data) diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c index 4ee23f4ebf4a..0b483bd0d3ca 100644 --- a/drivers/cpufreq/scmi-cpufreq.c +++ b/drivers/cpufreq/scmi-cpufreq.c @@ -144,6 +144,29 @@ scmi_get_cpu_power(struct device *cpu_dev, unsigned long *power, return 0; } +static int +scmi_get_rate_limit(u32 domain, bool has_fast_switch) +{ + int ret, rate_limit; + + if (has_fast_switch) { + /* + * Fast channels are used whenever available, + * so use their rate_limit value if populated. + */ + ret = perf_ops->fast_switch_rate_limit(ph, domain, + &rate_limit); + if (!ret && rate_limit) + return rate_limit; + } + + ret = perf_ops->rate_limit_get(ph, domain, &rate_limit); + if (ret) + return 0; + + return rate_limit; +} + static int scmi_cpufreq_init(struct cpufreq_policy *policy) { int ret, nr_opp, domain; @@ -250,6 +273,9 @@ static int scmi_cpufreq_init(struct cpufreq_policy *policy) policy->fast_switch_possible = perf_ops->fast_switch_possible(ph, domain); + policy->transition_delay_us = + scmi_get_rate_limit(domain, policy->fast_switch_possible); + return 0; out_free_opp: diff --git a/drivers/cpuidle/driver.c b/drivers/cpuidle/driver.c index d9cda7f6ccb9..cf5873cc45dc 100644 --- a/drivers/cpuidle/driver.c +++ b/drivers/cpuidle/driver.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "cpuidle.h" @@ -187,7 +188,7 @@ static void __cpuidle_driver_init(struct cpuidle_driver *drv) s->target_residency = div_u64(s->target_residency_ns, NSEC_PER_USEC); if (s->exit_latency > 0) - s->exit_latency_ns = s->exit_latency * NSEC_PER_USEC; + s->exit_latency_ns = mul_u32_u32(s->exit_latency, NSEC_PER_USEC); else if (s->exit_latency_ns < 0) s->exit_latency_ns = 0; else diff --git a/drivers/cpuidle/governors/haltpoll.c b/drivers/cpuidle/governors/haltpoll.c index 1dff3a52917d..663b7f164d20 100644 --- a/drivers/cpuidle/governors/haltpoll.c +++ b/drivers/cpuidle/governors/haltpoll.c @@ -98,10 +98,15 @@ static void adjust_poll_limit(struct cpuidle_device *dev, u64 block_ns) unsigned int shrink = guest_halt_poll_shrink; val = dev->poll_limit_ns; - if (shrink == 0) + if (shrink == 0) { val = 0; - else + } else { val /= shrink; + /* Reset value to 0 if shrunk below grow_start */ + if (val < guest_halt_poll_grow_start) + val = 0; + } + trace_guest_halt_poll_ns_shrink(val, dev->poll_limit_ns); dev->poll_limit_ns = val; } diff --git a/drivers/dax/super.c b/drivers/dax/super.c index f4b635526345..a0244f6bb44b 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -532,8 +532,7 @@ static int dax_fs_init(void) int rc; dax_cache = kmem_cache_create("dax_cache", sizeof(struct dax_device), 0, - (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_HWCACHE_ALIGN | SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, init_once); if (!dax_cache) return -ENOMEM; diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c index 34d77802c990..2709598f3008 100644 --- a/drivers/firmware/arm_scmi/driver.c +++ b/drivers/firmware/arm_scmi/driver.c @@ -1624,7 +1624,7 @@ static void scmi_common_fastchannel_init(const struct scmi_protocol_handle *ph, u8 describe_id, u32 message_id, u32 valid_size, u32 domain, void __iomem **p_addr, - struct scmi_fc_db_info **p_db) + struct scmi_fc_db_info **p_db, u32 *rate_limit) { int ret; u32 flags; @@ -1668,6 +1668,9 @@ scmi_common_fastchannel_init(const struct scmi_protocol_handle *ph, goto err_xfer; } + if (rate_limit) + *rate_limit = le32_to_cpu(resp->rate_limit) & GENMASK(19, 0); + phys_addr = le32_to_cpu(resp->chan_addr_low); phys_addr |= (u64)le32_to_cpu(resp->chan_addr_high) << 32; addr = devm_ioremap(ph->dev, phys_addr, size); diff --git a/drivers/firmware/arm_scmi/perf.c b/drivers/firmware/arm_scmi/perf.c index 981e327e63e3..8e832d1ad825 100644 --- a/drivers/firmware/arm_scmi/perf.c +++ b/drivers/firmware/arm_scmi/perf.c @@ -153,6 +153,7 @@ struct perf_dom_info { bool perf_fastchannels; bool level_indexing_mode; u32 opp_count; + u32 rate_limit_us; u32 sustained_freq_khz; u32 sustained_perf_level; unsigned long mult_factor; @@ -282,6 +283,8 @@ scmi_perf_domain_attributes_get(const struct scmi_protocol_handle *ph, if (PROTOCOL_REV_MAJOR(version) >= 0x4) dom_info->level_indexing_mode = SUPPORTS_LEVEL_INDEXING(flags); + dom_info->rate_limit_us = le32_to_cpu(attr->rate_limit_us) & + GENMASK(19, 0); dom_info->sustained_freq_khz = le32_to_cpu(attr->sustained_freq_khz); dom_info->sustained_perf_level = @@ -825,23 +828,27 @@ static void scmi_perf_domain_init_fc(const struct scmi_protocol_handle *ph, ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL, PERF_LEVEL_GET, 4, dom->id, - &fc[PERF_FC_LEVEL].get_addr, NULL); + &fc[PERF_FC_LEVEL].get_addr, NULL, + &fc[PERF_FC_LEVEL].rate_limit); ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL, PERF_LIMITS_GET, 8, dom->id, - &fc[PERF_FC_LIMIT].get_addr, NULL); + &fc[PERF_FC_LIMIT].get_addr, NULL, + &fc[PERF_FC_LIMIT].rate_limit); if (dom->info.set_perf) ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL, PERF_LEVEL_SET, 4, dom->id, &fc[PERF_FC_LEVEL].set_addr, - &fc[PERF_FC_LEVEL].set_db); + &fc[PERF_FC_LEVEL].set_db, + &fc[PERF_FC_LEVEL].rate_limit); if (dom->set_limits) ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL, PERF_LIMITS_SET, 8, dom->id, &fc[PERF_FC_LIMIT].set_addr, - &fc[PERF_FC_LIMIT].set_db); + &fc[PERF_FC_LIMIT].set_db, + &fc[PERF_FC_LIMIT].rate_limit); dom->fc_info = fc; } @@ -894,6 +901,23 @@ scmi_dvfs_transition_latency_get(const struct scmi_protocol_handle *ph, return dom->opp[dom->opp_count - 1].trans_latency_us * 1000; } +static int +scmi_dvfs_rate_limit_get(const struct scmi_protocol_handle *ph, + u32 domain, u32 *rate_limit) +{ + struct perf_dom_info *dom; + + if (!rate_limit) + return -EINVAL; + + dom = scmi_perf_domain_lookup(ph, domain); + if (IS_ERR(dom)) + return PTR_ERR(dom); + + *rate_limit = dom->rate_limit_us; + return 0; +} + static int scmi_dvfs_freq_set(const struct scmi_protocol_handle *ph, u32 domain, unsigned long freq, bool poll) { @@ -993,6 +1017,25 @@ static bool scmi_fast_switch_possible(const struct scmi_protocol_handle *ph, return dom->fc_info && dom->fc_info[PERF_FC_LEVEL].set_addr; } +static int scmi_fast_switch_rate_limit(const struct scmi_protocol_handle *ph, + u32 domain, u32 *rate_limit) +{ + struct perf_dom_info *dom; + + if (!rate_limit) + return -EINVAL; + + dom = scmi_perf_domain_lookup(ph, domain); + if (IS_ERR(dom)) + return PTR_ERR(dom); + + if (!dom->fc_info) + return -EINVAL; + + *rate_limit = dom->fc_info[PERF_FC_LEVEL].rate_limit; + return 0; +} + static enum scmi_power_scale scmi_power_scale_get(const struct scmi_protocol_handle *ph) { @@ -1009,11 +1052,13 @@ static const struct scmi_perf_proto_ops perf_proto_ops = { .level_set = scmi_perf_level_set, .level_get = scmi_perf_level_get, .transition_latency_get = scmi_dvfs_transition_latency_get, + .rate_limit_get = scmi_dvfs_rate_limit_get, .device_opps_add = scmi_dvfs_device_opps_add, .freq_set = scmi_dvfs_freq_set, .freq_get = scmi_dvfs_freq_get, .est_power_get = scmi_dvfs_est_power_get, .fast_switch_possible = scmi_fast_switch_possible, + .fast_switch_rate_limit = scmi_fast_switch_rate_limit, .power_scale_get = scmi_power_scale_get, }; diff --git a/drivers/firmware/arm_scmi/powercap.c b/drivers/firmware/arm_scmi/powercap.c index 2fab92367e42..ea9201e7044c 100644 --- a/drivers/firmware/arm_scmi/powercap.c +++ b/drivers/firmware/arm_scmi/powercap.c @@ -719,20 +719,24 @@ static void scmi_powercap_domain_init_fc(const struct scmi_protocol_handle *ph, ph->hops->fastchannel_init(ph, POWERCAP_DESCRIBE_FASTCHANNEL, POWERCAP_CAP_SET, 4, domain, &fc[POWERCAP_FC_CAP].set_addr, - &fc[POWERCAP_FC_CAP].set_db); + &fc[POWERCAP_FC_CAP].set_db, + &fc[POWERCAP_FC_CAP].rate_limit); ph->hops->fastchannel_init(ph, POWERCAP_DESCRIBE_FASTCHANNEL, POWERCAP_CAP_GET, 4, domain, - &fc[POWERCAP_FC_CAP].get_addr, NULL); + &fc[POWERCAP_FC_CAP].get_addr, NULL, + &fc[POWERCAP_FC_CAP].rate_limit); ph->hops->fastchannel_init(ph, POWERCAP_DESCRIBE_FASTCHANNEL, POWERCAP_PAI_SET, 4, domain, &fc[POWERCAP_FC_PAI].set_addr, - &fc[POWERCAP_FC_PAI].set_db); + &fc[POWERCAP_FC_PAI].set_db, + &fc[POWERCAP_FC_PAI].rate_limit); ph->hops->fastchannel_init(ph, POWERCAP_DESCRIBE_FASTCHANNEL, POWERCAP_PAI_GET, 4, domain, - &fc[POWERCAP_FC_PAI].get_addr, NULL); + &fc[POWERCAP_FC_PAI].get_addr, NULL, + &fc[POWERCAP_PAI_GET].rate_limit); *p_fc = fc; } diff --git a/drivers/firmware/arm_scmi/protocols.h b/drivers/firmware/arm_scmi/protocols.h index 693019fff0f6..317d3fb32676 100644 --- a/drivers/firmware/arm_scmi/protocols.h +++ b/drivers/firmware/arm_scmi/protocols.h @@ -235,6 +235,7 @@ struct scmi_fc_info { void __iomem *set_addr; void __iomem *get_addr; struct scmi_fc_db_info *set_db; + u32 rate_limit; }; /** @@ -273,7 +274,8 @@ struct scmi_proto_helpers_ops { u8 describe_id, u32 message_id, u32 valid_size, u32 domain, void __iomem **p_addr, - struct scmi_fc_db_info **p_db); + struct scmi_fc_db_info **p_db, + u32 *rate_limit); void (*fastchannel_db_ring)(struct scmi_fc_db_info *db); }; diff --git a/drivers/firmware/google/cbmem.c b/drivers/firmware/google/cbmem.c index 88e587ba1e0d..c2bffdc352a3 100644 --- a/drivers/firmware/google/cbmem.c +++ b/drivers/firmware/google/cbmem.c @@ -114,6 +114,12 @@ static int cbmem_entry_probe(struct coreboot_device *dev) return 0; } +static const struct coreboot_device_id cbmem_ids[] = { + { .tag = LB_TAG_CBMEM_ENTRY }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(coreboot, cbmem_ids); + static struct coreboot_driver cbmem_entry_driver = { .probe = cbmem_entry_probe, .drv = { @@ -121,7 +127,7 @@ static struct coreboot_driver cbmem_entry_driver = { .owner = THIS_MODULE, .dev_groups = dev_groups, }, - .tag = LB_TAG_CBMEM_ENTRY, + .id_table = cbmem_ids, }; module_coreboot_driver(cbmem_entry_driver); diff --git a/drivers/firmware/google/coreboot_table.c b/drivers/firmware/google/coreboot_table.c index 2a4469bf1b81..d4b6e581a6c6 100644 --- a/drivers/firmware/google/coreboot_table.c +++ b/drivers/firmware/google/coreboot_table.c @@ -28,8 +28,17 @@ static int coreboot_bus_match(struct device *dev, struct device_driver *drv) { struct coreboot_device *device = CB_DEV(dev); struct coreboot_driver *driver = CB_DRV(drv); + const struct coreboot_device_id *id; - return device->entry.tag == driver->tag; + if (!driver->id_table) + return 0; + + for (id = driver->id_table; id->tag; id++) { + if (device->entry.tag == id->tag) + return 1; + } + + return 0; } static int coreboot_bus_probe(struct device *dev) @@ -53,11 +62,20 @@ static void coreboot_bus_remove(struct device *dev) driver->remove(device); } -static struct bus_type coreboot_bus_type = { +static int coreboot_bus_uevent(const struct device *dev, struct kobj_uevent_env *env) +{ + struct coreboot_device *device = CB_DEV(dev); + u32 tag = device->entry.tag; + + return add_uevent_var(env, "MODALIAS=coreboot:t%08X", tag); +} + +static const struct bus_type coreboot_bus_type = { .name = "coreboot", .match = coreboot_bus_match, .probe = coreboot_bus_probe, .remove = coreboot_bus_remove, + .uevent = coreboot_bus_uevent, }; static void coreboot_device_release(struct device *dev) diff --git a/drivers/firmware/google/coreboot_table.h b/drivers/firmware/google/coreboot_table.h index d814dca33a08..86427989c57f 100644 --- a/drivers/firmware/google/coreboot_table.h +++ b/drivers/firmware/google/coreboot_table.h @@ -13,6 +13,7 @@ #define __COREBOOT_TABLE_H #include +#include /* Coreboot table header structure */ struct coreboot_table_header { @@ -93,7 +94,7 @@ struct coreboot_driver { int (*probe)(struct coreboot_device *); void (*remove)(struct coreboot_device *); struct device_driver drv; - u32 tag; + const struct coreboot_device_id *id_table; }; /* Register a driver that uses the data from a coreboot table. */ diff --git a/drivers/firmware/google/framebuffer-coreboot.c b/drivers/firmware/google/framebuffer-coreboot.c index 5c84bbebfef8..07c458bf64ec 100644 --- a/drivers/firmware/google/framebuffer-coreboot.c +++ b/drivers/firmware/google/framebuffer-coreboot.c @@ -80,13 +80,19 @@ static void framebuffer_remove(struct coreboot_device *dev) platform_device_unregister(pdev); } +static const struct coreboot_device_id framebuffer_ids[] = { + { .tag = CB_TAG_FRAMEBUFFER }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(coreboot, framebuffer_ids); + static struct coreboot_driver framebuffer_driver = { .probe = framebuffer_probe, .remove = framebuffer_remove, .drv = { .name = "framebuffer", }, - .tag = CB_TAG_FRAMEBUFFER, + .id_table = framebuffer_ids, }; module_coreboot_driver(framebuffer_driver); diff --git a/drivers/firmware/google/memconsole-coreboot.c b/drivers/firmware/google/memconsole-coreboot.c index 74b5286518ee..24c97a70aa80 100644 --- a/drivers/firmware/google/memconsole-coreboot.c +++ b/drivers/firmware/google/memconsole-coreboot.c @@ -96,13 +96,19 @@ static void memconsole_remove(struct coreboot_device *dev) memconsole_exit(); } +static const struct coreboot_device_id memconsole_ids[] = { + { .tag = CB_TAG_CBMEM_CONSOLE }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(coreboot, memconsole_ids); + static struct coreboot_driver memconsole_driver = { .probe = memconsole_probe, .remove = memconsole_remove, .drv = { .name = "memconsole", }, - .tag = CB_TAG_CBMEM_CONSOLE, + .id_table = memconsole_ids, }; module_coreboot_driver(memconsole_driver); diff --git a/drivers/firmware/google/vpd.c b/drivers/firmware/google/vpd.c index ee6e08c0592b..8e4216714b29 100644 --- a/drivers/firmware/google/vpd.c +++ b/drivers/firmware/google/vpd.c @@ -306,13 +306,19 @@ static void vpd_remove(struct coreboot_device *dev) kobject_put(vpd_kobj); } +static const struct coreboot_device_id vpd_ids[] = { + { .tag = CB_TAG_VPD }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(coreboot, vpd_ids); + static struct coreboot_driver vpd_driver = { .probe = vpd_probe, .remove = vpd_remove, .drv = { .name = "vpd", }, - .tag = CB_TAG_VPD, + .id_table = vpd_ids, }; module_coreboot_driver(vpd_driver); diff --git a/drivers/fpga/ice40-spi.c b/drivers/fpga/ice40-spi.c index 7cbb3558b844..c0028ae4c5b7 100644 --- a/drivers/fpga/ice40-spi.c +++ b/drivers/fpga/ice40-spi.c @@ -66,7 +66,7 @@ static int ice40_fpga_ops_write_init(struct fpga_manager *mgr, } /* Lock the bus, assert CRESET_B and SS_B and delay >200ns */ - spi_bus_lock(dev->master); + spi_bus_lock(dev->controller); gpiod_set_value(priv->reset, 1); @@ -94,7 +94,7 @@ static int ice40_fpga_ops_write_init(struct fpga_manager *mgr, ret = spi_sync_locked(dev, &message); fail: - spi_bus_unlock(dev->master); + spi_bus_unlock(dev->controller); return ret; } diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index 1301cec94f12..3fbb0bdb15c1 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -711,8 +711,9 @@ config GPIO_UNIPHIER Say yes here to support UniPhier GPIOs. config GPIO_VF610 - def_bool y - depends on ARCH_MXC + bool "VF610 GPIO support" + default y if SOC_VF610 + depends on ARCH_MXC || COMPILE_TEST select GPIOLIB_IRQCHIP help Say yes here to support i.MX or Vybrid vf610 GPIOs. @@ -1240,6 +1241,16 @@ config GPIO_BD9571MWV This driver can also be built as a module. If so, the module will be called gpio-bd9571mwv. +config GPIO_CROS_EC + tristate "ChromeOS EC GPIO support" + depends on CROS_EC + help + GPIO driver for the ChromeOS Embedded Controller (EC). GPIOs + cannot be set unless the system is unlocked. + + This driver can also be built as a module. If so, the module + will be called gpio-cros-ec. + config GPIO_CRYSTAL_COVE tristate "GPIO support for Crystal Cove PMIC" depends on (X86 || COMPILE_TEST) && INTEL_SOC_PMIC diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile index 9e40af196aae..7ae4d81de1df 100644 --- a/drivers/gpio/Makefile +++ b/drivers/gpio/Makefile @@ -46,6 +46,7 @@ obj-$(CONFIG_GPIO_BT8XX) += gpio-bt8xx.o obj-$(CONFIG_GPIO_CADENCE) += gpio-cadence.o obj-$(CONFIG_GPIO_CLPS711X) += gpio-clps711x.o obj-$(CONFIG_GPIO_SNPS_CREG) += gpio-creg-snps.o +obj-$(CONFIG_GPIO_CROS_EC) += gpio-cros-ec.o obj-$(CONFIG_GPIO_CRYSTAL_COVE) += gpio-crystalcove.o obj-$(CONFIG_GPIO_CS5535) += gpio-cs5535.o obj-$(CONFIG_GPIO_DA9052) += gpio-da9052.o diff --git a/drivers/gpio/gpio-cros-ec.c b/drivers/gpio/gpio-cros-ec.c new file mode 100644 index 000000000000..842e1c060414 --- /dev/null +++ b/drivers/gpio/gpio-cros-ec.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2024 Google LLC + * + * This driver provides the ability to control GPIOs on the Chrome OS EC. + * There isn't any direction control, and setting values on GPIOs is only + * possible when the system is unlocked. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Prefix all names to avoid collisions with EC <-> AP nets */ +static const char cros_ec_gpio_prefix[] = "EC:"; + +/* Setting gpios is only supported when the system is unlocked */ +static void cros_ec_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val) +{ + const char *name = gc->names[gpio] + strlen(cros_ec_gpio_prefix); + struct cros_ec_device *cros_ec = gpiochip_get_data(gc); + struct ec_params_gpio_set params = { + .val = val, + }; + int ret; + ssize_t copied; + + copied = strscpy(params.name, name, sizeof(params.name)); + if (copied < 0) + return; + + ret = cros_ec_cmd(cros_ec, 0, EC_CMD_GPIO_SET, ¶ms, + sizeof(params), NULL, 0); + if (ret < 0) + dev_err(gc->parent, "error setting gpio%d (%s) on EC: %d\n", gpio, name, ret); +} + +static int cros_ec_gpio_get(struct gpio_chip *gc, unsigned int gpio) +{ + const char *name = gc->names[gpio] + strlen(cros_ec_gpio_prefix); + struct cros_ec_device *cros_ec = gpiochip_get_data(gc); + struct ec_params_gpio_get params; + struct ec_response_gpio_get response; + int ret; + ssize_t copied; + + copied = strscpy(params.name, name, sizeof(params.name)); + if (copied < 0) + return -EINVAL; + + ret = cros_ec_cmd(cros_ec, 0, EC_CMD_GPIO_GET, ¶ms, + sizeof(params), &response, sizeof(response)); + if (ret < 0) { + dev_err(gc->parent, "error getting gpio%d (%s) on EC: %d\n", gpio, name, ret); + return ret; + } + + return response.val; +} + +#define CROS_EC_GPIO_INPUT BIT(8) +#define CROS_EC_GPIO_OUTPUT BIT(9) + +static int cros_ec_gpio_get_direction(struct gpio_chip *gc, unsigned int gpio) +{ + const char *name = gc->names[gpio] + strlen(cros_ec_gpio_prefix); + struct cros_ec_device *cros_ec = gpiochip_get_data(gc); + struct ec_params_gpio_get_v1 params = { + .subcmd = EC_GPIO_GET_INFO, + .get_info.index = gpio, + }; + struct ec_response_gpio_get_v1 response; + int ret; + + ret = cros_ec_cmd(cros_ec, 1, EC_CMD_GPIO_GET, ¶ms, + sizeof(params), &response, sizeof(response)); + if (ret < 0) { + dev_err(gc->parent, "error getting direction of gpio%d (%s) on EC: %d\n", gpio, name, ret); + return ret; + } + + if (response.get_info.flags & CROS_EC_GPIO_INPUT) + return GPIO_LINE_DIRECTION_IN; + + if (response.get_info.flags & CROS_EC_GPIO_OUTPUT) + return GPIO_LINE_DIRECTION_OUT; + + return -EINVAL; +} + +/* Query EC for all gpio line names */ +static int cros_ec_gpio_init_names(struct cros_ec_device *cros_ec, struct gpio_chip *gc) +{ + struct ec_params_gpio_get_v1 params = { + .subcmd = EC_GPIO_GET_INFO, + }; + struct ec_response_gpio_get_v1 response; + int ret, i; + /* EC may not NUL terminate */ + size_t name_len = strlen(cros_ec_gpio_prefix) + sizeof(response.get_info.name) + 1; + ssize_t copied; + const char **names; + char *str; + + names = devm_kcalloc(gc->parent, gc->ngpio, sizeof(*names), GFP_KERNEL); + if (!names) + return -ENOMEM; + gc->names = names; + + str = devm_kcalloc(gc->parent, gc->ngpio, name_len, GFP_KERNEL); + if (!str) + return -ENOMEM; + + /* Get gpio line names one at a time */ + for (i = 0; i < gc->ngpio; i++) { + params.get_info.index = i; + ret = cros_ec_cmd(cros_ec, 1, EC_CMD_GPIO_GET, ¶ms, + sizeof(params), &response, sizeof(response)); + if (ret < 0) { + dev_err_probe(gc->parent, ret, "error getting gpio%d info\n", i); + return ret; + } + + names[i] = str; + copied = scnprintf(str, name_len, "%s%s", cros_ec_gpio_prefix, + response.get_info.name); + if (copied < 0) + return copied; + + str += copied + 1; + } + + return 0; +} + +/* Query EC for number of gpios */ +static int cros_ec_gpio_ngpios(struct cros_ec_device *cros_ec) +{ + struct ec_params_gpio_get_v1 params = { + .subcmd = EC_GPIO_GET_COUNT, + }; + struct ec_response_gpio_get_v1 response; + int ret; + + ret = cros_ec_cmd(cros_ec, 1, EC_CMD_GPIO_GET, ¶ms, + sizeof(params), &response, sizeof(response)); + if (ret < 0) + return ret; + + return response.get_count.val; +} + +static int cros_ec_gpio_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device *parent = dev->parent; + struct cros_ec_dev *ec_dev = dev_get_drvdata(parent); + struct cros_ec_device *cros_ec = ec_dev->ec_dev; + struct gpio_chip *gc; + int ngpios; + int ret; + + /* Use the fwnode from the protocol device, e.g. cros-ec-spi */ + device_set_node(dev, dev_fwnode(cros_ec->dev)); + + ngpios = cros_ec_gpio_ngpios(cros_ec); + if (ngpios < 0) { + dev_err_probe(dev, ngpios, "error getting gpio count\n"); + return ngpios; + } + + gc = devm_kzalloc(dev, sizeof(*gc), GFP_KERNEL); + if (!gc) + return -ENOMEM; + + gc->ngpio = ngpios; + gc->parent = dev; + ret = cros_ec_gpio_init_names(cros_ec, gc); + if (ret) + return ret; + + gc->can_sleep = true; + gc->label = dev_name(dev); + gc->base = -1; + gc->set = cros_ec_gpio_set; + gc->get = cros_ec_gpio_get; + gc->get_direction = cros_ec_gpio_get_direction; + + return devm_gpiochip_add_data(dev, gc, cros_ec); +} + +static struct platform_driver cros_ec_gpio_driver = { + .probe = cros_ec_gpio_probe, + .driver = { + .name = "cros-ec-gpio", + }, +}; +module_platform_driver(cros_ec_gpio_driver); + +MODULE_DESCRIPTION("ChromeOS EC GPIO Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/gpio/gpio-eic-sprd.c b/drivers/gpio/gpio-eic-sprd.c index 806b88d8dfb7..2dd0e46c42ad 100644 --- a/drivers/gpio/gpio-eic-sprd.c +++ b/drivers/gpio/gpio-eic-sprd.c @@ -108,7 +108,6 @@ static struct sprd_eic *to_sprd_eic(struct notifier_block *nb) struct sprd_eic_variant_data { enum sprd_eic_type type; - u32 num_eics; }; static const char *sprd_eic_label_name[SPRD_EIC_MAX] = { @@ -118,22 +117,18 @@ static const char *sprd_eic_label_name[SPRD_EIC_MAX] = { static const struct sprd_eic_variant_data sc9860_eic_dbnc_data = { .type = SPRD_EIC_DEBOUNCE, - .num_eics = 8, }; static const struct sprd_eic_variant_data sc9860_eic_latch_data = { .type = SPRD_EIC_LATCH, - .num_eics = 8, }; static const struct sprd_eic_variant_data sc9860_eic_async_data = { .type = SPRD_EIC_ASYNC, - .num_eics = 8, }; static const struct sprd_eic_variant_data sc9860_eic_sync_data = { .type = SPRD_EIC_SYNC, - .num_eics = 8, }; static inline void __iomem *sprd_eic_offset_base(struct sprd_eic *sprd_eic, @@ -619,6 +614,7 @@ static int sprd_eic_probe(struct platform_device *pdev) struct gpio_irq_chip *irq; struct sprd_eic *sprd_eic; struct resource *res; + u16 num_banks = 0; int ret, i; pdata = of_device_get_match_data(dev); @@ -652,10 +648,12 @@ static int sprd_eic_probe(struct platform_device *pdev) sprd_eic->base[i] = devm_ioremap_resource(dev, res); if (IS_ERR(sprd_eic->base[i])) return PTR_ERR(sprd_eic->base[i]); + + num_banks++; } sprd_eic->chip.label = sprd_eic_label_name[sprd_eic->type]; - sprd_eic->chip.ngpio = pdata->num_eics; + sprd_eic->chip.ngpio = num_banks * SPRD_EIC_PER_BANK_NR; sprd_eic->chip.base = -1; sprd_eic->chip.parent = dev; sprd_eic->chip.direction_input = sprd_eic_direction_input; diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c index a13f3c18ccd4..8cfd3a89c018 100644 --- a/drivers/gpio/gpio-mvebu.c +++ b/drivers/gpio/gpio-mvebu.c @@ -99,7 +99,6 @@ struct mvebu_pwm { u32 offset; unsigned long clk_rate; struct gpio_desc *gpiod; - struct pwm_chip chip; spinlock_t lock; struct mvebu_gpio_chip *mvchip; @@ -615,7 +614,7 @@ static const struct regmap_config mvebu_gpio_regmap_config = { */ static struct mvebu_pwm *to_mvebu_pwm(struct pwm_chip *chip) { - return container_of(chip, struct mvebu_pwm, chip); + return pwmchip_get_drvdata(chip); } static int mvebu_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) @@ -789,6 +788,7 @@ static int mvebu_pwm_probe(struct platform_device *pdev, { struct device *dev = &pdev->dev; struct mvebu_pwm *mvpwm; + struct pwm_chip *chip; void __iomem *base; u32 offset; u32 set; @@ -813,9 +813,11 @@ static int mvebu_pwm_probe(struct platform_device *pdev, if (IS_ERR(mvchip->clk)) return PTR_ERR(mvchip->clk); - mvpwm = devm_kzalloc(dev, sizeof(struct mvebu_pwm), GFP_KERNEL); - if (!mvpwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(dev, mvchip->chip.ngpio, sizeof(*mvpwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + mvpwm = to_mvebu_pwm(chip); + mvchip->mvpwm = mvpwm; mvpwm->mvchip = mvchip; mvpwm->offset = offset; @@ -868,13 +870,11 @@ static int mvebu_pwm_probe(struct platform_device *pdev, return -EINVAL; } - mvpwm->chip.dev = dev; - mvpwm->chip.ops = &mvebu_pwm_ops; - mvpwm->chip.npwm = mvchip->chip.ngpio; + chip->ops = &mvebu_pwm_ops; spin_lock_init(&mvpwm->lock); - return devm_pwmchip_add(dev, &mvpwm->chip); + return devm_pwmchip_add(dev, chip); } #ifdef CONFIG_DEBUG_FS diff --git a/drivers/gpio/gpio-sim.c b/drivers/gpio/gpio-sim.c index c4106e37e6db..2ed5cbe7c8a8 100644 --- a/drivers/gpio/gpio-sim.c +++ b/drivers/gpio/gpio-sim.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -234,10 +235,10 @@ static void gpio_sim_dbg_show(struct seq_file *seq, struct gpio_chip *gc) guard(mutex)(&chip->lock); - for_each_requested_gpio(gc, i, label) + for_each_hwgpio(gc, i, label) seq_printf(seq, " gpio-%-3d (%s) %s,%s\n", gc->base + i, - label, + label ?: "", test_bit(i, chip->direction_map) ? "input" : test_bit(i, chip->value_map) ? "output-high" : "output-low", @@ -420,7 +421,7 @@ static int gpio_sim_add_bank(struct fwnode_handle *swnode, struct device *dev) ret = fwnode_property_read_string(swnode, "gpio-sim,label", &label); if (ret) { - label = devm_kasprintf(dev, GFP_KERNEL, "%s-%pfwP", + label = devm_kasprintf(dev, GFP_KERNEL, "%s:%pfwP", dev_name(dev), swnode); if (!label) return -ENOMEM; @@ -697,8 +698,10 @@ static struct gpio_sim_device *gpio_sim_hog_get_device(struct gpio_sim_hog *hog) return gpio_sim_line_get_device(line); } -static bool gpio_sim_device_is_live_unlocked(struct gpio_sim_device *dev) +static bool gpio_sim_device_is_live(struct gpio_sim_device *dev) { + lockdep_assert_held(&dev->lock); + return !!dev->pdev; } @@ -737,7 +740,7 @@ gpio_sim_device_config_live_show(struct config_item *item, char *page) bool live; scoped_guard(mutex, &dev->lock) - live = gpio_sim_device_is_live_unlocked(dev); + live = gpio_sim_device_is_live(dev); return sprintf(page, "%c\n", live ? '1' : '0'); } @@ -833,7 +836,7 @@ static int gpio_sim_add_hogs(struct gpio_sim_device *dev) GFP_KERNEL); else hog->chip_label = kasprintf(GFP_KERNEL, - "gpio-sim.%u-%pfwP", + "gpio-sim.%u:%pfwP", dev->id, bank->swnode); if (!hog->chip_label) { @@ -926,7 +929,7 @@ static bool gpio_sim_bank_labels_non_unique(struct gpio_sim_device *dev) return false; } -static int gpio_sim_device_activate_unlocked(struct gpio_sim_device *dev) +static int gpio_sim_device_activate(struct gpio_sim_device *dev) { struct platform_device_info pdevinfo; struct fwnode_handle *swnode; @@ -934,6 +937,8 @@ static int gpio_sim_device_activate_unlocked(struct gpio_sim_device *dev) struct gpio_sim_bank *bank; int ret; + lockdep_assert_held(&dev->lock); + if (list_empty(&dev->bank_list)) return -ENODATA; @@ -998,10 +1003,12 @@ static int gpio_sim_device_activate_unlocked(struct gpio_sim_device *dev) return 0; } -static void gpio_sim_device_deactivate_unlocked(struct gpio_sim_device *dev) +static void gpio_sim_device_deactivate(struct gpio_sim_device *dev) { struct fwnode_handle *swnode; + lockdep_assert_held(&dev->lock); + swnode = dev_fwnode(&dev->pdev->dev); platform_device_unregister(dev->pdev); gpio_sim_remove_hogs(dev); @@ -1023,12 +1030,12 @@ gpio_sim_device_config_live_store(struct config_item *item, guard(mutex)(&dev->lock); - if (live == gpio_sim_device_is_live_unlocked(dev)) + if (live == gpio_sim_device_is_live(dev)) ret = -EPERM; else if (live) - ret = gpio_sim_device_activate_unlocked(dev); + ret = gpio_sim_device_activate(dev); else - gpio_sim_device_deactivate_unlocked(dev); + gpio_sim_device_deactivate(dev); return ret ?: count; } @@ -1069,7 +1076,7 @@ static ssize_t gpio_sim_bank_config_chip_name_show(struct config_item *item, guard(mutex)(&dev->lock); - if (gpio_sim_device_is_live_unlocked(dev)) + if (gpio_sim_device_is_live(dev)) return device_for_each_child(&dev->pdev->dev, &ctx, gpio_sim_emit_chip_name); @@ -1098,7 +1105,7 @@ static ssize_t gpio_sim_bank_config_label_store(struct config_item *item, guard(mutex)(&dev->lock); - if (gpio_sim_device_is_live_unlocked(dev)) + if (gpio_sim_device_is_live(dev)) return -EBUSY; trimmed = gpio_sim_strdup_trimmed(page, count); @@ -1142,7 +1149,7 @@ gpio_sim_bank_config_num_lines_store(struct config_item *item, guard(mutex)(&dev->lock); - if (gpio_sim_device_is_live_unlocked(dev)) + if (gpio_sim_device_is_live(dev)) return -EBUSY; bank->num_lines = num_lines; @@ -1179,7 +1186,7 @@ static ssize_t gpio_sim_line_config_name_store(struct config_item *item, guard(mutex)(&dev->lock); - if (gpio_sim_device_is_live_unlocked(dev)) + if (gpio_sim_device_is_live(dev)) return -EBUSY; trimmed = gpio_sim_strdup_trimmed(page, count); @@ -1219,7 +1226,7 @@ static ssize_t gpio_sim_hog_config_name_store(struct config_item *item, guard(mutex)(&dev->lock); - if (gpio_sim_device_is_live_unlocked(dev)) + if (gpio_sim_device_is_live(dev)) return -EBUSY; trimmed = gpio_sim_strdup_trimmed(page, count); @@ -1274,7 +1281,7 @@ gpio_sim_hog_config_direction_store(struct config_item *item, guard(mutex)(&dev->lock); - if (gpio_sim_device_is_live_unlocked(dev)) + if (gpio_sim_device_is_live(dev)) return -EBUSY; if (sysfs_streq(page, "input")) @@ -1392,7 +1399,7 @@ gpio_sim_bank_config_make_line_group(struct config_group *group, guard(mutex)(&dev->lock); - if (gpio_sim_device_is_live_unlocked(dev)) + if (gpio_sim_device_is_live(dev)) return ERR_PTR(-EBUSY); line = kzalloc(sizeof(*line), GFP_KERNEL); @@ -1445,7 +1452,7 @@ gpio_sim_device_config_make_bank_group(struct config_group *group, guard(mutex)(&dev->lock); - if (gpio_sim_device_is_live_unlocked(dev)) + if (gpio_sim_device_is_live(dev)) return ERR_PTR(-EBUSY); bank = kzalloc(sizeof(*bank), GFP_KERNEL); @@ -1467,8 +1474,8 @@ static void gpio_sim_device_config_group_release(struct config_item *item) struct gpio_sim_device *dev = to_gpio_sim_device(item); scoped_guard(mutex, &dev->lock) { - if (gpio_sim_device_is_live_unlocked(dev)) - gpio_sim_device_deactivate_unlocked(dev); + if (gpio_sim_device_is_live(dev)) + gpio_sim_device_deactivate(dev); } mutex_destroy(&dev->lock); diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c index cd3e9657cc36..7f140df40f35 100644 --- a/drivers/gpio/gpiolib-acpi.c +++ b/drivers/gpio/gpiolib-acpi.c @@ -126,7 +126,7 @@ static DEFINE_MUTEX(acpi_gpio_deferred_req_irqs_lock); static LIST_HEAD(acpi_gpio_deferred_req_irqs_list); static bool acpi_gpio_deferred_req_irqs_done; -static int acpi_gpiochip_find(struct gpio_chip *gc, void *data) +static int acpi_gpiochip_find(struct gpio_chip *gc, const void *data) { return device_match_acpi_handle(&gc->gpiodev->dev, data); } @@ -1402,17 +1402,17 @@ static int acpi_find_gpio_count(struct acpi_resource *ares, void *data) } /** - * acpi_gpio_count - count the GPIOs associated with a device / function - * @dev: GPIO consumer, can be %NULL for system-global GPIOs + * acpi_gpio_count - count the GPIOs associated with a firmware node / function + * @fwnode: firmware node of the GPIO consumer * @con_id: function within the GPIO consumer * * Return: - * The number of GPIOs associated with a device / function or %-ENOENT, + * The number of GPIOs associated with a firmware node / function or %-ENOENT, * if no GPIO has been assigned to the requested function. */ -int acpi_gpio_count(struct device *dev, const char *con_id) +int acpi_gpio_count(const struct fwnode_handle *fwnode, const char *con_id) { - struct acpi_device *adev = ACPI_COMPANION(dev); + struct acpi_device *adev = to_acpi_device_node(fwnode); const union acpi_object *obj; const struct acpi_gpio_mapping *gm; int count = -ENOENT; @@ -1429,8 +1429,7 @@ int acpi_gpio_count(struct device *dev, const char *con_id) snprintf(propname, sizeof(propname), "%s", gpio_suffixes[i]); - ret = acpi_dev_get_property(adev, propname, ACPI_TYPE_ANY, - &obj); + ret = acpi_dev_get_property(adev, propname, ACPI_TYPE_ANY, &obj); if (ret == 0) { if (obj->type == ACPI_TYPE_LOCAL_REFERENCE) count = 1; diff --git a/drivers/gpio/gpiolib-acpi.h b/drivers/gpio/gpiolib-acpi.h index 0fcd7e14d7f9..7e1c51d04040 100644 --- a/drivers/gpio/gpiolib-acpi.h +++ b/drivers/gpio/gpiolib-acpi.h @@ -33,7 +33,7 @@ struct gpio_desc *acpi_find_gpio(struct fwnode_handle *fwnode, enum gpiod_flags *dflags, unsigned long *lookupflags); -int acpi_gpio_count(struct device *dev, const char *con_id); +int acpi_gpio_count(const struct fwnode_handle *fwnode, const char *con_id); #else static inline void acpi_gpiochip_add(struct gpio_chip *chip) { } static inline void acpi_gpiochip_remove(struct gpio_chip *chip) { } @@ -51,7 +51,8 @@ acpi_find_gpio(struct fwnode_handle *fwnode, const char *con_id, { return ERR_PTR(-ENOENT); } -static inline int acpi_gpio_count(struct device *dev, const char *con_id) +static inline int acpi_gpio_count(const struct fwnode_handle *fwnode, + const char *con_id) { return -ENODEV; } diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index 2a88736629ef..f384fa278764 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include @@ -61,11 +60,6 @@ static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_values), 8)); * interface to gpiolib GPIOs via ioctl()s. */ -typedef __poll_t (*poll_fn)(struct file *, struct poll_table_struct *); -typedef long (*ioctl_fn)(struct file *, unsigned int, unsigned long); -typedef ssize_t (*read_fn)(struct file *, char __user *, - size_t count, loff_t *); - /* * GPIO line handle management */ @@ -210,9 +204,9 @@ static long linehandle_ioctl(struct file *file, unsigned int cmd, unsigned int i; int ret; - guard(rwsem_read)(&lh->gdev->sem); + guard(srcu)(&lh->gdev->srcu); - if (!lh->gdev->chip) + if (!rcu_access_pointer(lh->gdev->chip)) return -ENODEV; switch (cmd) { @@ -337,7 +331,7 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip) /* Request each GPIO */ for (i = 0; i < handlereq.lines; i++) { u32 offset = handlereq.lineoffsets[i]; - struct gpio_desc *desc = gpiochip_get_desc(gdev->chip, offset); + struct gpio_desc *desc = gpio_device_get_desc(gdev, offset); if (IS_ERR(desc)) { ret = PTR_ERR(desc); @@ -1525,9 +1519,9 @@ static long linereq_ioctl(struct file *file, unsigned int cmd, struct linereq *lr = file->private_data; void __user *ip = (void __user *)arg; - guard(rwsem_read)(&lr->gdev->sem); + guard(srcu)(&lr->gdev->srcu); - if (!lr->gdev->chip) + if (!rcu_access_pointer(lr->gdev->chip)) return -ENODEV; switch (cmd) { @@ -1556,9 +1550,9 @@ static __poll_t linereq_poll(struct file *file, struct linereq *lr = file->private_data; __poll_t events = 0; - guard(rwsem_read)(&lr->gdev->sem); + guard(srcu)(&lr->gdev->srcu); - if (!lr->gdev->chip) + if (!rcu_access_pointer(lr->gdev->chip)) return EPOLLHUP | EPOLLERR; poll_wait(file, &lr->wait, wait); @@ -1578,9 +1572,9 @@ static ssize_t linereq_read(struct file *file, char __user *buf, ssize_t bytes_read = 0; int ret; - guard(rwsem_read)(&lr->gdev->sem); + guard(srcu)(&lr->gdev->srcu); - if (!lr->gdev->chip) + if (!rcu_access_pointer(lr->gdev->chip)) return -ENODEV; if (count < sizeof(le)) @@ -1744,7 +1738,7 @@ static int linereq_create(struct gpio_device *gdev, void __user *ip) /* Request each GPIO */ for (i = 0; i < ulr.num_lines; i++) { u32 offset = ulr.offsets[i]; - struct gpio_desc *desc = gpiochip_get_desc(gdev->chip, offset); + struct gpio_desc *desc = gpio_device_get_desc(gdev, offset); if (IS_ERR(desc)) { ret = PTR_ERR(desc); @@ -1879,9 +1873,9 @@ static __poll_t lineevent_poll(struct file *file, struct lineevent_state *le = file->private_data; __poll_t events = 0; - guard(rwsem_read)(&le->gdev->sem); + guard(srcu)(&le->gdev->srcu); - if (!le->gdev->chip) + if (!rcu_access_pointer(le->gdev->chip)) return EPOLLHUP | EPOLLERR; poll_wait(file, &le->wait, wait); @@ -1917,9 +1911,9 @@ static ssize_t lineevent_read(struct file *file, char __user *buf, ssize_t ge_size; int ret; - guard(rwsem_read)(&le->gdev->sem); + guard(srcu)(&le->gdev->srcu); - if (!le->gdev->chip) + if (!rcu_access_pointer(le->gdev->chip)) return -ENODEV; /* @@ -2000,9 +1994,9 @@ static long lineevent_ioctl(struct file *file, unsigned int cmd, void __user *ip = (void __user *)arg; struct gpiohandle_data ghd; - guard(rwsem_read)(&le->gdev->sem); + guard(srcu)(&le->gdev->srcu); - if (!le->gdev->chip) + if (!rcu_access_pointer(le->gdev->chip)) return -ENODEV; /* @@ -2128,7 +2122,7 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip) lflags = eventreq.handleflags; eflags = eventreq.eventflags; - desc = gpiochip_get_desc(gdev->chip, offset); + desc = gpio_device_get_desc(gdev, offset); if (IS_ERR(desc)) return PTR_ERR(desc); @@ -2300,21 +2294,26 @@ static void gpio_v2_line_info_changed_to_v1( static void gpio_desc_to_lineinfo(struct gpio_desc *desc, struct gpio_v2_line_info *info) { - struct gpio_chip *gc = desc->gdev->chip; unsigned long dflags; + const char *label; + + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return; memset(info, 0, sizeof(*info)); info->offset = gpio_chip_hwgpio(desc); - scoped_guard(spinlock_irqsave, &gpio_lock) { - if (desc->name) - strscpy(info->name, desc->name, sizeof(info->name)); + if (desc->name) + strscpy(info->name, desc->name, sizeof(info->name)); - if (desc->label) - strscpy(info->consumer, desc->label, + dflags = READ_ONCE(desc->flags); + + scoped_guard(srcu, &desc->srcu) { + label = gpiod_get_label(desc); + if (label && test_bit(FLAG_REQUESTED, &dflags)) + strscpy(info->consumer, label, sizeof(info->consumer)); - - dflags = READ_ONCE(desc->flags); } /* @@ -2334,8 +2333,8 @@ static void gpio_desc_to_lineinfo(struct gpio_desc *desc, test_bit(FLAG_USED_AS_IRQ, &dflags) || test_bit(FLAG_EXPORT, &dflags) || test_bit(FLAG_SYSFS, &dflags) || - !gpiochip_line_is_valid(gc, info->offset) || - !pinctrl_gpio_can_use_line(gc, info->offset)) + !gpiochip_line_is_valid(guard.gc, info->offset) || + !pinctrl_gpio_can_use_line(guard.gc, info->offset)) info->flags |= GPIO_V2_LINE_FLAG_USED; if (test_bit(FLAG_IS_OUT, &dflags)) @@ -2422,7 +2421,7 @@ static int lineinfo_get_v1(struct gpio_chardev_data *cdev, void __user *ip, return -EFAULT; /* this doubles as a range check on line_offset */ - desc = gpiochip_get_desc(cdev->gdev->chip, lineinfo.line_offset); + desc = gpio_device_get_desc(cdev->gdev, lineinfo.line_offset); if (IS_ERR(desc)) return PTR_ERR(desc); @@ -2459,7 +2458,7 @@ static int lineinfo_get(struct gpio_chardev_data *cdev, void __user *ip, if (memchr_inv(lineinfo.padding, 0, sizeof(lineinfo.padding))) return -EINVAL; - desc = gpiochip_get_desc(cdev->gdev->chip, lineinfo.offset); + desc = gpio_device_get_desc(cdev->gdev, lineinfo.offset); if (IS_ERR(desc)) return PTR_ERR(desc); @@ -2508,10 +2507,10 @@ static long gpio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) struct gpio_device *gdev = cdev->gdev; void __user *ip = (void __user *)arg; - guard(rwsem_read)(&gdev->sem); + guard(srcu)(&gdev->srcu); /* We fail any subsequent ioctl():s when the chip is gone */ - if (!gdev->chip) + if (!rcu_access_pointer(gdev->chip)) return -ENODEV; /* Fill in the struct and pass to userspace */ @@ -2594,9 +2593,9 @@ static __poll_t lineinfo_watch_poll(struct file *file, struct gpio_chardev_data *cdev = file->private_data; __poll_t events = 0; - guard(rwsem_read)(&cdev->gdev->sem); + guard(srcu)(&cdev->gdev->srcu); - if (!cdev->gdev->chip) + if (!rcu_access_pointer(cdev->gdev->chip)) return EPOLLHUP | EPOLLERR; poll_wait(file, &cdev->wait, pollt); @@ -2617,9 +2616,9 @@ static ssize_t lineinfo_watch_read(struct file *file, char __user *buf, int ret; size_t event_size; - guard(rwsem_read)(&cdev->gdev->sem); + guard(srcu)(&cdev->gdev->srcu); - if (!cdev->gdev->chip) + if (!rcu_access_pointer(cdev->gdev->chip)) return -ENODEV; #ifndef CONFIG_GPIO_CDEV_V1 @@ -2694,17 +2693,17 @@ static int gpio_chrdev_open(struct inode *inode, struct file *file) struct gpio_chardev_data *cdev; int ret = -ENOMEM; - guard(rwsem_read)(&gdev->sem); + guard(srcu)(&gdev->srcu); /* Fail on open if the backing gpiochip is gone */ - if (!gdev->chip) + if (!rcu_access_pointer(gdev->chip)) return -ENODEV; cdev = kzalloc(sizeof(*cdev), GFP_KERNEL); if (!cdev) return -ENODEV; - cdev->watched_lines = bitmap_zalloc(gdev->chip->ngpio, GFP_KERNEL); + cdev->watched_lines = bitmap_zalloc(gdev->ngpio, GFP_KERNEL); if (!cdev->watched_lines) goto out_free_cdev; @@ -2784,6 +2783,7 @@ static const struct file_operations gpio_fileops = { int gpiolib_cdev_register(struct gpio_device *gdev, dev_t devt) { + struct gpio_chip *gc; int ret; cdev_init(&gdev->chrdev, &gpio_fileops); @@ -2794,8 +2794,12 @@ int gpiolib_cdev_register(struct gpio_device *gdev, dev_t devt) if (ret) return ret; - chip_dbg(gdev->chip, "added GPIO chardev (%d:%d)\n", - MAJOR(devt), gdev->id); + guard(srcu)(&gdev->srcu); + gc = srcu_dereference(gdev->chip, &gdev->srcu); + if (!gc) + return -ENODEV; + + chip_dbg(gc, "added GPIO chardev (%d:%d)\n", MAJOR(devt), gdev->id); return 0; } diff --git a/drivers/gpio/gpiolib-devres.c b/drivers/gpio/gpiolib-devres.c index fe9ce6b19f15..4987e62dcb3d 100644 --- a/drivers/gpio/gpiolib-devres.c +++ b/drivers/gpio/gpiolib-devres.c @@ -158,7 +158,7 @@ struct gpio_desc *devm_fwnode_gpiod_get_index(struct device *dev, if (!dr) return ERR_PTR(-ENOMEM); - desc = fwnode_gpiod_get_index(fwnode, con_id, index, flags, label); + desc = gpiod_find_and_request(dev, fwnode, con_id, index, flags, label, false); if (IS_ERR(desc)) { devres_free(dr); return desc; diff --git a/drivers/gpio/gpiolib-legacy.c b/drivers/gpio/gpiolib-legacy.c index 97f4b498e343..b138682fec3d 100644 --- a/drivers/gpio/gpiolib-legacy.c +++ b/drivers/gpio/gpiolib-legacy.c @@ -6,6 +6,9 @@ #include "gpiolib.h" +/* + * **DEPRECATED** This function is deprecated and must not be used in new code. + */ void gpio_free(unsigned gpio) { gpiod_free(gpio_to_desc(gpio)); @@ -17,6 +20,8 @@ EXPORT_SYMBOL_GPL(gpio_free); * @gpio: the GPIO number * @flags: GPIO configuration as specified by GPIOF_* * @label: a literal description string of this GPIO + * + * **DEPRECATED** This function is deprecated and must not be used in new code. */ int gpio_request_one(unsigned gpio, unsigned long flags, const char *label) { @@ -53,6 +58,9 @@ int gpio_request_one(unsigned gpio, unsigned long flags, const char *label) } EXPORT_SYMBOL_GPL(gpio_request_one); +/* + * **DEPRECATED** This function is deprecated and must not be used in new code. + */ int gpio_request(unsigned gpio, const char *label) { struct gpio_desc *desc = gpio_to_desc(gpio); @@ -69,6 +77,8 @@ EXPORT_SYMBOL_GPL(gpio_request); * gpio_request_array - request multiple GPIOs in a single call * @array: array of the 'struct gpio' * @num: how many GPIOs in the array + * + * **DEPRECATED** This function is deprecated and must not be used in new code. */ int gpio_request_array(const struct gpio *array, size_t num) { @@ -92,6 +102,8 @@ EXPORT_SYMBOL_GPL(gpio_request_array); * gpio_free_array - release multiple GPIOs in a single call * @array: array of the 'struct gpio' * @num: how many GPIOs in the array + * + * **DEPRECATED** This function is deprecated and must not be used in new code. */ void gpio_free_array(const struct gpio *array, size_t num) { diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index de5759546417..647ea3394fe4 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -68,7 +68,7 @@ static int of_gpio_named_count(const struct device_node *np, /** * of_gpio_spi_cs_get_count() - special GPIO counting for SPI - * @dev: Consuming device + * @np: Consuming device node * @con_id: Function within the GPIO consumer * * Some elder GPIO controllers need special quirks. Currently we handle @@ -78,10 +78,9 @@ static int of_gpio_named_count(const struct device_node *np, * the counting of "cs-gpios" to count "gpios" transparent to the * driver. */ -static int of_gpio_spi_cs_get_count(struct device *dev, const char *con_id) +static int of_gpio_spi_cs_get_count(const struct device_node *np, + const char *con_id) { - struct device_node *np = dev->of_node; - if (!IS_ENABLED(CONFIG_SPI_MASTER)) return 0; if (!con_id || strcmp(con_id, "cs")) @@ -93,13 +92,14 @@ static int of_gpio_spi_cs_get_count(struct device *dev, const char *con_id) return of_gpio_named_count(np, "gpios"); } -int of_gpio_get_count(struct device *dev, const char *con_id) +int of_gpio_count(const struct fwnode_handle *fwnode, const char *con_id) { + const struct device_node *np = to_of_node(fwnode); int ret; char propname[32]; unsigned int i; - ret = of_gpio_spi_cs_get_count(dev, con_id); + ret = of_gpio_spi_cs_get_count(np, con_id); if (ret > 0) return ret; @@ -111,16 +111,17 @@ int of_gpio_get_count(struct device *dev, const char *con_id) snprintf(propname, sizeof(propname), "%s", gpio_suffixes[i]); - ret = of_gpio_named_count(dev->of_node, propname); + ret = of_gpio_named_count(np, propname); if (ret > 0) break; } return ret ? ret : -ENOENT; } -static int of_gpiochip_match_node_and_xlate(struct gpio_chip *chip, void *data) +static int of_gpiochip_match_node_and_xlate(struct gpio_chip *chip, + const void *data) { - struct of_phandle_args *gpiospec = data; + const struct of_phandle_args *gpiospec = data; return device_match_of_node(&chip->gpiodev->dev, gpiospec->np) && chip->of_xlate && @@ -128,7 +129,7 @@ static int of_gpiochip_match_node_and_xlate(struct gpio_chip *chip, void *data) } static struct gpio_device * -of_find_gpio_device_by_xlate(struct of_phandle_args *gpiospec) +of_find_gpio_device_by_xlate(const struct of_phandle_args *gpiospec) { return gpio_device_find(gpiospec, of_gpiochip_match_node_and_xlate); } @@ -414,6 +415,8 @@ out: * @propname: Name of property containing gpio specifier(s) * @index: index of the GPIO * + * **DEPRECATED** This function is deprecated and must not be used in new code. + * * Returns GPIO number to use with Linux generic GPIO API, or one of the errno * value on the error condition. */ @@ -799,7 +802,7 @@ static int of_gpiochip_add_hog(struct gpio_chip *chip, struct device_node *hog) return ret; #ifdef CONFIG_OF_DYNAMIC - desc->hog = hog; + WRITE_ONCE(desc->hog, hog); #endif } @@ -847,11 +850,11 @@ static void of_gpiochip_remove_hog(struct gpio_chip *chip, struct gpio_desc *desc; for_each_gpio_desc_with_flag(chip, desc, FLAG_IS_HOGGED) - if (desc->hog == hog) + if (READ_ONCE(desc->hog) == hog) gpiochip_free_own_desc(desc); } -static int of_gpiochip_match_node(struct gpio_chip *chip, void *data) +static int of_gpiochip_match_node(struct gpio_chip *chip, const void *data) { return device_match_of_node(&chip->gpiodev->dev, data); } diff --git a/drivers/gpio/gpiolib-of.h b/drivers/gpio/gpiolib-of.h index 6b3a5347c5d9..16d6ac8cb156 100644 --- a/drivers/gpio/gpiolib-of.h +++ b/drivers/gpio/gpiolib-of.h @@ -9,6 +9,7 @@ #include struct device; +struct fwnode_handle; struct gpio_chip; struct gpio_desc; @@ -21,7 +22,7 @@ struct gpio_desc *of_find_gpio(struct device_node *np, unsigned long *lookupflags); int of_gpiochip_add(struct gpio_chip *gc); void of_gpiochip_remove(struct gpio_chip *gc); -int of_gpio_get_count(struct device *dev, const char *con_id); +int of_gpio_count(const struct fwnode_handle *fwnode, const char *con_id); #else static inline struct gpio_desc *of_find_gpio(struct device_node *np, const char *con_id, @@ -32,7 +33,8 @@ static inline struct gpio_desc *of_find_gpio(struct device_node *np, } static inline int of_gpiochip_add(struct gpio_chip *gc) { return 0; } static inline void of_gpiochip_remove(struct gpio_chip *gc) { } -static inline int of_gpio_get_count(struct device *dev, const char *con_id) +static inline int of_gpio_count(const struct fwnode_handle *fwnode, + const char *con_id) { return 0; } diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index 6bf5332136e5..6853ecd98bcb 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #include #include @@ -13,6 +14,7 @@ #include #include #include +#include #include #include @@ -170,6 +172,10 @@ static int gpio_sysfs_request_irq(struct device *dev, unsigned char flags) unsigned long irq_flags; int ret; + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + data->irq = gpiod_to_irq(desc); if (data->irq < 0) return -EIO; @@ -194,7 +200,7 @@ static int gpio_sysfs_request_irq(struct device *dev, unsigned char flags) * Remove this redundant call (along with the corresponding * unlock) when those drivers have been fixed. */ - ret = gpiochip_lock_as_irq(desc->gdev->chip, gpio_chip_hwgpio(desc)); + ret = gpiochip_lock_as_irq(guard.gc, gpio_chip_hwgpio(desc)); if (ret < 0) goto err_put_kn; @@ -208,7 +214,7 @@ static int gpio_sysfs_request_irq(struct device *dev, unsigned char flags) return 0; err_unlock: - gpiochip_unlock_as_irq(desc->gdev->chip, gpio_chip_hwgpio(desc)); + gpiochip_unlock_as_irq(guard.gc, gpio_chip_hwgpio(desc)); err_put_kn: sysfs_put(data->value_kn); @@ -224,9 +230,13 @@ static void gpio_sysfs_free_irq(struct device *dev) struct gpiod_data *data = dev_get_drvdata(dev); struct gpio_desc *desc = data->desc; + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return; + data->irq_flags = 0; free_irq(data->irq, data); - gpiochip_unlock_as_irq(desc->gdev->chip, gpio_chip_hwgpio(desc)); + gpiochip_unlock_as_irq(guard.gc, gpio_chip_hwgpio(desc)); sysfs_put(data->value_kn); } @@ -400,27 +410,27 @@ static const struct attribute_group *gpio_groups[] = { static ssize_t base_show(struct device *dev, struct device_attribute *attr, char *buf) { - const struct gpio_chip *chip = dev_get_drvdata(dev); + const struct gpio_device *gdev = dev_get_drvdata(dev); - return sysfs_emit(buf, "%d\n", chip->base); + return sysfs_emit(buf, "%d\n", gdev->base); } static DEVICE_ATTR_RO(base); static ssize_t label_show(struct device *dev, struct device_attribute *attr, char *buf) { - const struct gpio_chip *chip = dev_get_drvdata(dev); + const struct gpio_device *gdev = dev_get_drvdata(dev); - return sysfs_emit(buf, "%s\n", chip->label ?: ""); + return sysfs_emit(buf, "%s\n", gdev->label); } static DEVICE_ATTR_RO(label); static ssize_t ngpio_show(struct device *dev, struct device_attribute *attr, char *buf) { - const struct gpio_chip *chip = dev_get_drvdata(dev); + const struct gpio_device *gdev = dev_get_drvdata(dev); - return sysfs_emit(buf, "%u\n", chip->ngpio); + return sysfs_emit(buf, "%u\n", gdev->ngpio); } static DEVICE_ATTR_RO(ngpio); @@ -443,13 +453,12 @@ static ssize_t export_store(const struct class *class, const char *buf, size_t len) { struct gpio_desc *desc; - struct gpio_chip *gc; int status, offset; long gpio; status = kstrtol(buf, 0, &gpio); - if (status < 0) - goto done; + if (status) + return status; desc = gpio_to_desc(gpio); /* reject invalid GPIOs */ @@ -457,9 +466,13 @@ static ssize_t export_store(const struct class *class, pr_warn("%s: invalid GPIO %ld\n", __func__, gpio); return -EINVAL; } - gc = desc->gdev->chip; + + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + offset = gpio_chip_hwgpio(desc); - if (!gpiochip_line_is_valid(gc, offset)) { + if (!gpiochip_line_is_valid(guard.gc, offset)) { pr_warn("%s: GPIO %ld masked\n", __func__, gpio); return -EINVAL; } @@ -562,8 +575,6 @@ int gpiod_export(struct gpio_desc *desc, bool direction_may_change) const char *ioname = NULL; struct gpio_device *gdev; struct gpiod_data *data; - struct gpio_chip *chip; - unsigned long flags; struct device *dev; int status, offset; @@ -578,29 +589,28 @@ int gpiod_export(struct gpio_desc *desc, bool direction_may_change) return -EINVAL; } + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + + if (test_and_set_bit(FLAG_EXPORT, &desc->flags)) + return -EPERM; + gdev = desc->gdev; - chip = gdev->chip; mutex_lock(&sysfs_lock); /* check if chip is being removed */ - if (!chip || !gdev->mockdev) { + if (!gdev->mockdev) { status = -ENODEV; goto err_unlock; } - spin_lock_irqsave(&gpio_lock, flags); - if (!test_bit(FLAG_REQUESTED, &desc->flags) || - test_bit(FLAG_EXPORT, &desc->flags)) { - spin_unlock_irqrestore(&gpio_lock, flags); - gpiod_dbg(desc, "%s: unavailable (requested=%d, exported=%d)\n", - __func__, - test_bit(FLAG_REQUESTED, &desc->flags), - test_bit(FLAG_EXPORT, &desc->flags)); + if (!test_bit(FLAG_REQUESTED, &desc->flags)) { + gpiod_dbg(desc, "%s: unavailable (not requested)\n", __func__); status = -EPERM; goto err_unlock; } - spin_unlock_irqrestore(&gpio_lock, flags); data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) { @@ -610,14 +620,14 @@ int gpiod_export(struct gpio_desc *desc, bool direction_may_change) data->desc = desc; mutex_init(&data->mutex); - if (chip->direction_input && chip->direction_output) + if (guard.gc->direction_input && guard.gc->direction_output) data->direction_can_change = direction_may_change; else data->direction_can_change = false; offset = gpio_chip_hwgpio(desc); - if (chip->names && chip->names[offset]) - ioname = chip->names[offset]; + if (guard.gc->names && guard.gc->names[offset]) + ioname = guard.gc->names[offset]; dev = device_create_with_groups(&gpio_class, &gdev->dev, MKDEV(0, 0), data, gpio_groups, @@ -628,7 +638,6 @@ int gpiod_export(struct gpio_desc *desc, bool direction_may_change) goto err_free_data; } - set_bit(FLAG_EXPORT, &desc->flags); mutex_unlock(&sysfs_lock); return 0; @@ -636,6 +645,7 @@ err_free_data: kfree(data); err_unlock: mutex_unlock(&sysfs_lock); + clear_bit(FLAG_EXPORT, &desc->flags); gpiod_dbg(desc, "%s: status %d\n", __func__, status); return status; } @@ -732,7 +742,7 @@ EXPORT_SYMBOL_GPL(gpiod_unexport); int gpiochip_sysfs_register(struct gpio_device *gdev) { - struct gpio_chip *chip = gdev->chip; + struct gpio_chip *chip; struct device *parent; struct device *dev; @@ -745,6 +755,12 @@ int gpiochip_sysfs_register(struct gpio_device *gdev) if (!class_is_registered(&gpio_class)) return 0; + guard(srcu)(&gdev->srcu); + + chip = srcu_dereference(gdev->chip, &gdev->srcu); + if (!chip) + return -ENODEV; + /* * For sysfs backward compatibility we need to preserve this * preferred parenting to the gpio_chip parent field, if set. @@ -755,7 +771,7 @@ int gpiochip_sysfs_register(struct gpio_device *gdev) parent = &gdev->dev; /* use chip->base for the ID; it's already known to be unique */ - dev = device_create_with_groups(&gpio_class, parent, MKDEV(0, 0), chip, + dev = device_create_with_groups(&gpio_class, parent, MKDEV(0, 0), gdev, gpiochip_groups, GPIOCHIP_NAME "%d", chip->base); if (IS_ERR(dev)) @@ -771,18 +787,24 @@ int gpiochip_sysfs_register(struct gpio_device *gdev) void gpiochip_sysfs_unregister(struct gpio_device *gdev) { struct gpio_desc *desc; - struct gpio_chip *chip = gdev->chip; + struct gpio_chip *chip; - if (!gdev->mockdev) + scoped_guard(mutex, &sysfs_lock) { + if (!gdev->mockdev) + return; + + device_unregister(gdev->mockdev); + + /* prevent further gpiod exports */ + gdev->mockdev = NULL; + } + + guard(srcu)(&gdev->srcu); + + chip = srcu_dereference(gdev->chip, &gdev->srcu); + if (!chip) return; - device_unregister(gdev->mockdev); - - /* prevent further gpiod exports */ - mutex_lock(&sysfs_lock); - gdev->mockdev = NULL; - mutex_unlock(&sysfs_lock); - /* unregister gpiod class devices owned by sysfs */ for_each_gpio_desc_with_flag(chip, desc, FLAG_SYSFS) { gpiod_unexport(desc); @@ -790,11 +812,29 @@ void gpiochip_sysfs_unregister(struct gpio_device *gdev) } } +/* + * We're not really looking for a device - we just want to iterate over the + * list and call this callback for each GPIO device. This is why this function + * always returns 0. + */ +static int gpiofind_sysfs_register(struct gpio_chip *gc, const void *data) +{ + struct gpio_device *gdev = gc->gpiodev; + int ret; + + if (gdev->mockdev) + return 0; + + ret = gpiochip_sysfs_register(gdev); + if (ret) + chip_err(gc, "failed to register the sysfs entry: %d\n", ret); + + return 0; +} + static int __init gpiolib_sysfs_init(void) { - int status; - unsigned long flags; - struct gpio_device *gdev; + int status; status = class_register(&gpio_class); if (status < 0) @@ -806,26 +846,8 @@ static int __init gpiolib_sysfs_init(void) * We run before arch_initcall() so chip->dev nodes can have * registered, and so arch_initcall() can always gpiod_export(). */ - spin_lock_irqsave(&gpio_lock, flags); - list_for_each_entry(gdev, &gpio_devices, list) { - if (gdev->mockdev) - continue; + (void)gpio_device_find(NULL, gpiofind_sysfs_register); - /* - * TODO we yield gpio_lock here because - * gpiochip_sysfs_register() acquires a mutex. This is unsafe - * and needs to be fixed. - * - * Also it would be nice to use gpio_device_find() here so we - * can keep gpio_chips local to gpiolib.c, but the yield of - * gpio_lock prevents us from doing this. - */ - spin_unlock_irqrestore(&gpio_lock, flags); - status = gpiochip_sysfs_register(gdev); - spin_lock_irqsave(&gpio_lock, flags); - } - spin_unlock_irqrestore(&gpio_lock, flags); - - return status; + return 0; } postcore_initcall(gpiolib_sysfs_init); diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 75be4a3ca7f8..ce94e37bcbee 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -14,12 +15,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include @@ -63,7 +66,7 @@ static int gpio_bus_match(struct device *dev, struct device_driver *drv) return 1; } -static struct bus_type gpio_bus_type = { +static const struct bus_type gpio_bus_type = { .name = "gpio", .match = gpio_bus_match, }; @@ -73,15 +76,14 @@ static struct bus_type gpio_bus_type = { */ #define FASTPATH_NGPIO CONFIG_GPIOLIB_FASTPATH_LIMIT -/* gpio_lock prevents conflicts during gpio_desc[] table updates. - * While any GPIO is requested, its gpio_chip is not removable; - * each GPIO's "requested" flag serves as a lock and refcount. - */ -DEFINE_SPINLOCK(gpio_lock); - static DEFINE_MUTEX(gpio_lookup_lock); static LIST_HEAD(gpio_lookup_list); -LIST_HEAD(gpio_devices); + +static LIST_HEAD(gpio_devices); +/* Protects the GPIO device list against concurrent modifications. */ +static DEFINE_MUTEX(gpio_devices_lock); +/* Ensures coherence during read-only accesses to the list of GPIO devices. */ +DEFINE_STATIC_SRCU(gpio_devices_srcu); static DEFINE_MUTEX(gpio_machine_hogs_mutex); static LIST_HEAD(gpio_machine_hogs); @@ -97,9 +99,34 @@ static void gpiochip_irqchip_free_valid_mask(struct gpio_chip *gc); static bool gpiolib_initialized; -static inline void desc_set_label(struct gpio_desc *d, const char *label) +const char *gpiod_get_label(struct gpio_desc *desc) { - d->label = label; + unsigned long flags; + + flags = READ_ONCE(desc->flags); + if (test_bit(FLAG_USED_AS_IRQ, &flags) && + !test_bit(FLAG_REQUESTED, &flags)) + return "interrupt"; + + return test_bit(FLAG_REQUESTED, &flags) ? + srcu_dereference(desc->label, &desc->srcu) : NULL; +} + +static int desc_set_label(struct gpio_desc *desc, const char *label) +{ + const char *new = NULL, *old; + + if (label) { + new = kstrdup_const(label, GFP_KERNEL); + if (!new) + return -ENOMEM; + } + + old = rcu_replace_pointer(desc->label, new, 1); + synchronize_srcu(&desc->srcu); + kfree_const(old); + + return 0; } /** @@ -113,20 +140,16 @@ static inline void desc_set_label(struct gpio_desc *d, const char *label) struct gpio_desc *gpio_to_desc(unsigned gpio) { struct gpio_device *gdev; - unsigned long flags; - spin_lock_irqsave(&gpio_lock, flags); - - list_for_each_entry(gdev, &gpio_devices, list) { - if (gdev->base <= gpio && - gdev->base + gdev->ngpio > gpio) { - spin_unlock_irqrestore(&gpio_lock, flags); - return &gdev->descs[gpio - gdev->base]; + scoped_guard(srcu, &gpio_devices_srcu) { + list_for_each_entry_srcu(gdev, &gpio_devices, list, + srcu_read_lock_held(&gpio_devices_srcu)) { + if (gdev->base <= gpio && + gdev->base + gdev->ngpio > gpio) + return &gdev->descs[gpio - gdev->base]; } } - spin_unlock_irqrestore(&gpio_lock, flags); - if (!gpio_is_valid(gpio)) pr_warn("invalid GPIO %d\n", gpio); @@ -161,16 +184,6 @@ EXPORT_SYMBOL_GPL(gpiochip_get_desc); struct gpio_desc * gpio_device_get_desc(struct gpio_device *gdev, unsigned int hwnum) { - struct gpio_chip *gc; - - /* - * FIXME: This will be locked once we protect gdev->chip everywhere - * with SRCU. - */ - gc = gdev->chip; - if (!gc) - return ERR_PTR(-ENODEV); - if (hwnum >= gdev->ngpio) return ERR_PTR(-EINVAL); @@ -198,12 +211,18 @@ EXPORT_SYMBOL_GPL(desc_to_gpio); /** * gpiod_to_chip - Return the GPIO chip to which a GPIO descriptor belongs * @desc: descriptor to return the chip of + * + * *DEPRECATED* + * This function is unsafe and should not be used. Using the chip address + * without taking the SRCU read lock may result in dereferencing a dangling + * pointer. */ struct gpio_chip *gpiod_to_chip(const struct gpio_desc *desc) { - if (!desc || !desc->gdev) + if (!desc) return NULL; - return desc->gdev->chip; + + return gpio_device_get_chip(desc->gdev); } EXPORT_SYMBOL_GPL(gpiod_to_chip); @@ -262,6 +281,7 @@ EXPORT_SYMBOL(gpio_device_get_label); * Returns: * Address of the GPIO chip backing this device. * + * *DEPRECATED* * Until we can get rid of all non-driver users of struct gpio_chip, we must * provide a way of retrieving the pointer to it from struct gpio_device. This * is *NOT* safe as the GPIO API is considered to be hot-unpluggable and the @@ -272,7 +292,7 @@ EXPORT_SYMBOL(gpio_device_get_label); */ struct gpio_chip *gpio_device_get_chip(struct gpio_device *gdev) { - return gdev->chip; + return rcu_dereference_check(gdev->chip, 1); } EXPORT_SYMBOL_GPL(gpio_device_get_chip); @@ -282,7 +302,8 @@ static int gpiochip_find_base_unlocked(int ngpio) struct gpio_device *gdev; int base = GPIO_DYNAMIC_BASE; - list_for_each_entry(gdev, &gpio_devices, list) { + list_for_each_entry_srcu(gdev, &gpio_devices, list, + lockdep_is_held(&gpio_devices_lock)) { /* found a free space? */ if (gdev->base >= base + ngpio) break; @@ -311,25 +332,36 @@ static int gpiochip_find_base_unlocked(int ngpio) */ int gpiod_get_direction(struct gpio_desc *desc) { - struct gpio_chip *gc; + unsigned long flags; unsigned int offset; int ret; - gc = gpiod_to_chip(desc); + /* + * We cannot use VALIDATE_DESC() as we must not return 0 for a NULL + * descriptor like we usually do. + */ + if (!desc || IS_ERR(desc)) + return -EINVAL; + + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + offset = gpio_chip_hwgpio(desc); + flags = READ_ONCE(desc->flags); /* * Open drain emulation using input mode may incorrectly report * input here, fix that up. */ - if (test_bit(FLAG_OPEN_DRAIN, &desc->flags) && - test_bit(FLAG_IS_OUT, &desc->flags)) + if (test_bit(FLAG_OPEN_DRAIN, &flags) && + test_bit(FLAG_IS_OUT, &flags)) return 0; - if (!gc->get_direction) + if (!guard.gc->get_direction) return -ENOTSUPP; - ret = gc->get_direction(gc, offset); + ret = guard.gc->get_direction(guard.gc, offset); if (ret < 0) return ret; @@ -337,7 +369,8 @@ int gpiod_get_direction(struct gpio_desc *desc) if (ret > 0) ret = 1; - assign_bit(FLAG_IS_OUT, &desc->flags, !ret); + assign_bit(FLAG_IS_OUT, &flags, !ret); + WRITE_ONCE(desc->flags, flags); return ret; } @@ -354,23 +387,25 @@ static int gpiodev_add_to_list_unlocked(struct gpio_device *gdev) { struct gpio_device *prev, *next; + lockdep_assert_held(&gpio_devices_lock); + if (list_empty(&gpio_devices)) { /* initial entry in list */ - list_add_tail(&gdev->list, &gpio_devices); + list_add_tail_rcu(&gdev->list, &gpio_devices); return 0; } next = list_first_entry(&gpio_devices, struct gpio_device, list); if (gdev->base + gdev->ngpio <= next->base) { /* add before first entry */ - list_add(&gdev->list, &gpio_devices); + list_add_rcu(&gdev->list, &gpio_devices); return 0; } prev = list_last_entry(&gpio_devices, struct gpio_device, list); if (prev->base + prev->ngpio <= gdev->base) { /* add behind last entry */ - list_add_tail(&gdev->list, &gpio_devices); + list_add_tail_rcu(&gdev->list, &gpio_devices); return 0; } @@ -382,11 +417,13 @@ static int gpiodev_add_to_list_unlocked(struct gpio_device *gdev) /* add between prev and next */ if (prev->base + prev->ngpio <= gdev->base && gdev->base + gdev->ngpio <= next->base) { - list_add(&gdev->list, &prev->list); + list_add_rcu(&gdev->list, &prev->list); return 0; } } + synchronize_srcu(&gpio_devices_srcu); + return -EBUSY; } @@ -399,26 +436,28 @@ static int gpiodev_add_to_list_unlocked(struct gpio_device *gdev) static struct gpio_desc *gpio_name_to_desc(const char * const name) { struct gpio_device *gdev; - unsigned long flags; + struct gpio_desc *desc; + struct gpio_chip *gc; if (!name) return NULL; - spin_lock_irqsave(&gpio_lock, flags); + guard(srcu)(&gpio_devices_srcu); - list_for_each_entry(gdev, &gpio_devices, list) { - struct gpio_desc *desc; + list_for_each_entry_srcu(gdev, &gpio_devices, list, + srcu_read_lock_held(&gpio_devices_srcu)) { + guard(srcu)(&gdev->srcu); - for_each_gpio_desc(gdev->chip, desc) { - if (desc->name && !strcmp(desc->name, name)) { - spin_unlock_irqrestore(&gpio_lock, flags); + gc = srcu_dereference(gdev->chip, &gdev->srcu); + if (!gc) + continue; + + for_each_gpio_desc(gc, desc) { + if (desc->name && !strcmp(desc->name, name)) return desc; - } } } - spin_unlock_irqrestore(&gpio_lock, flags); - return NULL; } @@ -656,13 +695,23 @@ EXPORT_SYMBOL_GPL(gpiochip_line_is_valid); static void gpiodev_release(struct device *dev) { struct gpio_device *gdev = to_gpio_device(dev); + unsigned int i; + + for (i = 0; i < gdev->ngpio; i++) + cleanup_srcu_struct(&gdev->descs[i].srcu); ida_free(&gpio_ida, gdev->id); kfree_const(gdev->label); kfree(gdev->descs); + cleanup_srcu_struct(&gdev->srcu); kfree(gdev); } +static const struct device_type gpio_dev_type = { + .name = "gpio_chip", + .release = gpiodev_release, +}; + #ifdef CONFIG_GPIO_CDEV #define gcdev_register(gdev, devt) gpiolib_cdev_register((gdev), (devt)) #define gcdev_unregister(gdev) gpiolib_cdev_unregister((gdev)) @@ -680,6 +729,8 @@ static int gpiochip_setup_dev(struct gpio_device *gdev) struct fwnode_handle *fwnode = dev_fwnode(&gdev->dev); int ret; + device_initialize(&gdev->dev); + /* * If fwnode doesn't belong to another device, it's safe to clear its * initialized flag. @@ -691,15 +742,12 @@ static int gpiochip_setup_dev(struct gpio_device *gdev) if (ret) return ret; - /* From this point, the .release() function cleans up gpio_device */ - gdev->dev.release = gpiodev_release; - ret = gpiochip_sysfs_register(gdev); if (ret) goto err_remove_device; dev_dbg(&gdev->dev, "registered GPIOs %d to %d on %s\n", gdev->base, - gdev->base + gdev->ngpio - 1, gdev->chip->label ? : "generic"); + gdev->base + gdev->ngpio - 1, gdev->label); return 0; @@ -720,9 +768,6 @@ static void gpiochip_machine_hog(struct gpio_chip *gc, struct gpiod_hog *hog) return; } - if (test_bit(FLAG_IS_HOGGED, &desc->flags)) - return; - rv = gpiod_hog(desc, hog->line_name, hog->lflags, hog->dflags); if (rv) gpiod_err(desc, "%s: unable to hog GPIO line (%s:%u): %d\n", @@ -748,7 +793,10 @@ static void gpiochip_setup_devs(void) struct gpio_device *gdev; int ret; - list_for_each_entry(gdev, &gpio_devices, list) { + guard(srcu)(&gpio_devices_srcu); + + list_for_each_entry_srcu(gdev, &gpio_devices, list, + srcu_read_lock_held(&gpio_devices_srcu)) { ret = gpiochip_setup_dev(gdev); if (ret) dev_err(&gdev->dev, @@ -813,8 +861,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, struct lock_class_key *request_key) { struct gpio_device *gdev; - unsigned long flags; - unsigned int i; + unsigned int desc_index; int base = 0; int ret = 0; @@ -825,9 +872,11 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, gdev = kzalloc(sizeof(*gdev), GFP_KERNEL); if (!gdev) return -ENOMEM; + + gdev->dev.type = &gpio_dev_type; gdev->dev.bus = &gpio_bus_type; gdev->dev.parent = gc->parent; - gdev->chip = gc; + rcu_assign_pointer(gdev->chip, gc); gc->gpiodev = gdev; gpiochip_set_data(gc, data); @@ -851,7 +900,6 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, if (ret) goto err_free_ida; - device_initialize(&gdev->dev); if (gc->parent && gc->parent->driver) gdev->owner = gc->parent->driver->owner; else if (gc->owner) @@ -877,53 +925,55 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, } gdev->ngpio = gc->ngpio; + gdev->can_sleep = gc->can_sleep; - spin_lock_irqsave(&gpio_lock, flags); - - /* - * TODO: this allocates a Linux GPIO number base in the global - * GPIO numberspace for this chip. In the long run we want to - * get *rid* of this numberspace and use only descriptors, but - * it may be a pipe dream. It will not happen before we get rid - * of the sysfs interface anyways. - */ - base = gc->base; - if (base < 0) { - base = gpiochip_find_base_unlocked(gc->ngpio); + scoped_guard(mutex, &gpio_devices_lock) { + /* + * TODO: this allocates a Linux GPIO number base in the global + * GPIO numberspace for this chip. In the long run we want to + * get *rid* of this numberspace and use only descriptors, but + * it may be a pipe dream. It will not happen before we get rid + * of the sysfs interface anyways. + */ + base = gc->base; if (base < 0) { - spin_unlock_irqrestore(&gpio_lock, flags); - ret = base; - base = 0; + base = gpiochip_find_base_unlocked(gc->ngpio); + if (base < 0) { + ret = base; + base = 0; + goto err_free_label; + } + + /* + * TODO: it should not be necessary to reflect the + * assigned base outside of the GPIO subsystem. Go over + * drivers and see if anyone makes use of this, else + * drop this and assign a poison instead. + */ + gc->base = base; + } else { + dev_warn(&gdev->dev, + "Static allocation of GPIO base is deprecated, use dynamic allocation.\n"); + } + + gdev->base = base; + + ret = gpiodev_add_to_list_unlocked(gdev); + if (ret) { + chip_err(gc, "GPIO integer space overlap, cannot add chip\n"); goto err_free_label; } - /* - * TODO: it should not be necessary to reflect the assigned - * base outside of the GPIO subsystem. Go over drivers and - * see if anyone makes use of this, else drop this and assign - * a poison instead. - */ - gc->base = base; - } else { - dev_warn(&gdev->dev, - "Static allocation of GPIO base is deprecated, use dynamic allocation.\n"); - } - gdev->base = base; - - ret = gpiodev_add_to_list_unlocked(gdev); - if (ret) { - spin_unlock_irqrestore(&gpio_lock, flags); - chip_err(gc, "GPIO integer space overlap, cannot add chip\n"); - goto err_free_label; } - for (i = 0; i < gc->ngpio; i++) - gdev->descs[i].gdev = gdev; - - spin_unlock_irqrestore(&gpio_lock, flags); + for (desc_index = 0; desc_index < gc->ngpio; desc_index++) + gdev->descs[desc_index].gdev = gdev; BLOCKING_INIT_NOTIFIER_HEAD(&gdev->line_state_notifier); BLOCKING_INIT_NOTIFIER_HEAD(&gdev->device_notifier); - init_rwsem(&gdev->sem); + + ret = init_srcu_struct(&gdev->srcu); + if (ret) + goto err_remove_from_list; #ifdef CONFIG_PINCTRL INIT_LIST_HEAD(&gdev->pin_ranges); @@ -932,32 +982,36 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, if (gc->names) { ret = gpiochip_set_desc_names(gc); if (ret) - goto err_remove_from_list; + goto err_cleanup_gdev_srcu; } ret = gpiochip_set_names(gc); if (ret) - goto err_remove_from_list; + goto err_cleanup_gdev_srcu; ret = gpiochip_init_valid_mask(gc); if (ret) - goto err_remove_from_list; + goto err_cleanup_gdev_srcu; - ret = of_gpiochip_add(gc); - if (ret) - goto err_free_gpiochip_mask; + for (desc_index = 0; desc_index < gc->ngpio; desc_index++) { + struct gpio_desc *desc = &gdev->descs[desc_index]; - for (i = 0; i < gc->ngpio; i++) { - struct gpio_desc *desc = &gdev->descs[i]; + ret = init_srcu_struct(&desc->srcu); + if (ret) + goto err_cleanup_desc_srcu; - if (gc->get_direction && gpiochip_line_is_valid(gc, i)) { + if (gc->get_direction && gpiochip_line_is_valid(gc, desc_index)) { assign_bit(FLAG_IS_OUT, - &desc->flags, !gc->get_direction(gc, i)); + &desc->flags, !gc->get_direction(gc, desc_index)); } else { assign_bit(FLAG_IS_OUT, &desc->flags, !gc->direction_input); } } + ret = of_gpiochip_add(gc); + if (ret) + goto err_cleanup_desc_srcu; + ret = gpiochip_add_pin_ranges(gc); if (ret) goto err_remove_of_chip; @@ -1003,12 +1057,16 @@ err_free_hogs: gpiochip_remove_pin_ranges(gc); err_remove_of_chip: of_gpiochip_remove(gc); -err_free_gpiochip_mask: +err_cleanup_desc_srcu: + while (desc_index--) + cleanup_srcu_struct(&gdev->descs[desc_index].srcu); gpiochip_free_valid_mask(gc); +err_cleanup_gdev_srcu: + cleanup_srcu_struct(&gdev->srcu); err_remove_from_list: - spin_lock_irqsave(&gpio_lock, flags); - list_del(&gdev->list); - spin_unlock_irqrestore(&gpio_lock, flags); + scoped_guard(mutex, &gpio_devices_lock) + list_del_rcu(&gdev->list); + synchronize_srcu(&gpio_devices_srcu); if (gdev->dev.release) { /* release() has been registered by gpiochip_setup_dev() */ gpio_device_put(gdev); @@ -1044,16 +1102,18 @@ EXPORT_SYMBOL_GPL(gpiochip_add_data_with_key); void gpiochip_remove(struct gpio_chip *gc) { struct gpio_device *gdev = gc->gpiodev; - unsigned long flags; - unsigned int i; - - down_write(&gdev->sem); /* FIXME: should the legacy sysfs handling be moved to gpio_device? */ gpiochip_sysfs_unregister(gdev); gpiochip_free_hogs(gc); + + scoped_guard(mutex, &gpio_devices_lock) + list_del_rcu(&gdev->list); + synchronize_srcu(&gpio_devices_srcu); + /* Numb the device, cancelling all outstanding operations */ - gdev->chip = NULL; + rcu_assign_pointer(gdev->chip, NULL); + synchronize_srcu(&gdev->srcu); gpiochip_irqchip_remove(gc); acpi_gpiochip_remove(gc); of_gpiochip_remove(gc); @@ -1065,20 +1125,6 @@ void gpiochip_remove(struct gpio_chip *gc) */ gpiochip_set_data(gc, NULL); - spin_lock_irqsave(&gpio_lock, flags); - for (i = 0; i < gdev->ngpio; i++) { - if (test_bit(FLAG_REQUESTED, &gdev->descs[i].flags)) - break; - } - spin_unlock_irqrestore(&gpio_lock, flags); - - if (i != gdev->ngpio) - dev_crit(&gdev->dev, - "REMOVING GPIOCHIP WITH GPIOS STILL REQUESTED\n"); - - scoped_guard(spinlock_irqsave, &gpio_lock) - list_del(&gdev->list); - /* * The gpiochip side puts its use of the device to rest here: * if there are no userspace clients, the chardev and device will @@ -1086,7 +1132,6 @@ void gpiochip_remove(struct gpio_chip *gc) * gone. */ gcdev_unregister(gdev); - up_write(&gdev->sem); gpio_device_put(gdev); } EXPORT_SYMBOL_GPL(gpiochip_remove); @@ -1112,11 +1157,12 @@ EXPORT_SYMBOL_GPL(gpiochip_remove); * If the function returns non-NULL, the returned reference must be freed by * the caller using gpio_device_put(). */ -struct gpio_device *gpio_device_find(void *data, +struct gpio_device *gpio_device_find(const void *data, int (*match)(struct gpio_chip *gc, - void *data)) + const void *data)) { struct gpio_device *gdev; + struct gpio_chip *gc; /* * Not yet but in the future the spinlock below will become a mutex. @@ -1125,10 +1171,15 @@ struct gpio_device *gpio_device_find(void *data, */ might_sleep(); - guard(spinlock_irqsave)(&gpio_lock); + guard(srcu)(&gpio_devices_srcu); - list_for_each_entry(gdev, &gpio_devices, list) { - if (gdev->chip && match(gdev->chip, data)) + list_for_each_entry_srcu(gdev, &gpio_devices, list, + srcu_read_lock_held(&gpio_devices_srcu)) { + guard(srcu)(&gdev->srcu); + + gc = srcu_dereference(gdev->chip, &gdev->srcu); + + if (gc && match(gc, data)) return gpio_device_get(gdev); } @@ -1136,7 +1187,7 @@ struct gpio_device *gpio_device_find(void *data, } EXPORT_SYMBOL_GPL(gpio_device_find); -static int gpio_chip_match_by_label(struct gpio_chip *gc, void *label) +static int gpio_chip_match_by_label(struct gpio_chip *gc, const void *label) { return gc->label && !strcmp(gc->label, label); } @@ -1156,7 +1207,7 @@ struct gpio_device *gpio_device_find_by_label(const char *label) } EXPORT_SYMBOL_GPL(gpio_device_find_by_label); -static int gpio_chip_match_by_fwnode(struct gpio_chip *gc, void *fwnode) +static int gpio_chip_match_by_fwnode(struct gpio_chip *gc, const void *fwnode) { return device_match_fwnode(&gc->gpiodev->dev, fwnode); } @@ -1254,8 +1305,8 @@ static void gpiochip_irqchip_free_valid_mask(struct gpio_chip *gc) gpiochip_free_mask(&gc->irq.valid_mask); } -bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gc, - unsigned int offset) +static bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gc, + unsigned int offset) { if (!gpiochip_line_is_valid(gc, offset)) return false; @@ -1264,7 +1315,6 @@ bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gc, return true; return test_bit(offset, gc->irq.valid_mask); } -EXPORT_SYMBOL_GPL(gpiochip_irqchip_irq_valid); #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY @@ -1439,6 +1489,43 @@ static unsigned int gpiochip_child_offset_to_irq_noop(struct gpio_chip *gc, return offset; } +/** + * gpiochip_irq_domain_activate() - Lock a GPIO to be used as an IRQ + * @domain: The IRQ domain used by this IRQ chip + * @data: Outermost irq_data associated with the IRQ + * @reserve: If set, only reserve an interrupt vector instead of assigning one + * + * This function is a wrapper that calls gpiochip_lock_as_irq() and is to be + * used as the activate function for the &struct irq_domain_ops. The host_data + * for the IRQ domain must be the &struct gpio_chip. + */ +static int gpiochip_irq_domain_activate(struct irq_domain *domain, + struct irq_data *data, bool reserve) +{ + struct gpio_chip *gc = domain->host_data; + unsigned int hwirq = irqd_to_hwirq(data); + + return gpiochip_lock_as_irq(gc, hwirq); +} + +/** + * gpiochip_irq_domain_deactivate() - Unlock a GPIO used as an IRQ + * @domain: The IRQ domain used by this IRQ chip + * @data: Outermost irq_data associated with the IRQ + * + * This function is a wrapper that will call gpiochip_unlock_as_irq() and is to + * be used as the deactivate function for the &struct irq_domain_ops. The + * host_data for the IRQ domain must be the &struct gpio_chip. + */ +static void gpiochip_irq_domain_deactivate(struct irq_domain *domain, + struct irq_data *data) +{ + struct gpio_chip *gc = domain->host_data; + unsigned int hwirq = irqd_to_hwirq(data); + + return gpiochip_unlock_as_irq(gc, hwirq); +} + static void gpiochip_hierarchy_setup_domain_ops(struct irq_domain_ops *ops) { ops->activate = gpiochip_irq_domain_activate; @@ -1556,7 +1643,8 @@ static bool gpiochip_hierarchy_is_hierarchical(struct gpio_chip *gc) * gpiochip by assigning the gpiochip as chip data, and using the irqchip * stored inside the gpiochip. */ -int gpiochip_irq_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hwirq) +static int gpiochip_irq_map(struct irq_domain *d, unsigned int irq, + irq_hw_number_t hwirq) { struct gpio_chip *gc = d->host_data; int ret = 0; @@ -1593,9 +1681,8 @@ int gpiochip_irq_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hwi return 0; } -EXPORT_SYMBOL_GPL(gpiochip_irq_map); -void gpiochip_irq_unmap(struct irq_domain *d, unsigned int irq) +static void gpiochip_irq_unmap(struct irq_domain *d, unsigned int irq) { struct gpio_chip *gc = d->host_data; @@ -1604,7 +1691,6 @@ void gpiochip_irq_unmap(struct irq_domain *d, unsigned int irq) irq_set_chip_and_handler(irq, NULL, NULL); irq_set_chip_data(irq, NULL); } -EXPORT_SYMBOL_GPL(gpiochip_irq_unmap); static const struct irq_domain_ops gpiochip_domain_ops = { .map = gpiochip_irq_map, @@ -1626,50 +1712,6 @@ static struct irq_domain *gpiochip_simple_create_domain(struct gpio_chip *gc) return domain; } -/* - * TODO: move these activate/deactivate in under the hierarchicial - * irqchip implementation as static once SPMI and SSBI (all external - * users) are phased over. - */ -/** - * gpiochip_irq_domain_activate() - Lock a GPIO to be used as an IRQ - * @domain: The IRQ domain used by this IRQ chip - * @data: Outermost irq_data associated with the IRQ - * @reserve: If set, only reserve an interrupt vector instead of assigning one - * - * This function is a wrapper that calls gpiochip_lock_as_irq() and is to be - * used as the activate function for the &struct irq_domain_ops. The host_data - * for the IRQ domain must be the &struct gpio_chip. - */ -int gpiochip_irq_domain_activate(struct irq_domain *domain, - struct irq_data *data, bool reserve) -{ - struct gpio_chip *gc = domain->host_data; - unsigned int hwirq = irqd_to_hwirq(data); - - return gpiochip_lock_as_irq(gc, hwirq); -} -EXPORT_SYMBOL_GPL(gpiochip_irq_domain_activate); - -/** - * gpiochip_irq_domain_deactivate() - Unlock a GPIO used as an IRQ - * @domain: The IRQ domain used by this IRQ chip - * @data: Outermost irq_data associated with the IRQ - * - * This function is a wrapper that will call gpiochip_unlock_as_irq() and is to - * be used as the deactivate function for the &struct irq_domain_ops. The - * host_data for the IRQ domain must be the &struct gpio_chip. - */ -void gpiochip_irq_domain_deactivate(struct irq_domain *domain, - struct irq_data *data) -{ - struct gpio_chip *gc = domain->host_data; - unsigned int hwirq = irqd_to_hwirq(data); - - return gpiochip_unlock_as_irq(gc, hwirq); -} -EXPORT_SYMBOL_GPL(gpiochip_irq_domain_deactivate); - static int gpiochip_to_irq(struct gpio_chip *gc, unsigned int offset) { struct irq_domain *domain = gc->irq.domain; @@ -2189,58 +2231,41 @@ EXPORT_SYMBOL_GPL(gpiochip_remove_pin_ranges); */ static int gpiod_request_commit(struct gpio_desc *desc, const char *label) { - struct gpio_chip *gc = desc->gdev->chip; - unsigned long flags; unsigned int offset; int ret; - if (label) { - label = kstrdup_const(label, GFP_KERNEL); - if (!label) - return -ENOMEM; - } + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; - spin_lock_irqsave(&gpio_lock, flags); + if (test_and_set_bit(FLAG_REQUESTED, &desc->flags)) + return -EBUSY; /* NOTE: gpio_request() can be called in early boot, * before IRQs are enabled, for non-sleeping (SOC) GPIOs. */ - if (test_and_set_bit(FLAG_REQUESTED, &desc->flags) == 0) { - desc_set_label(desc, label ? : "?"); - } else { - ret = -EBUSY; - goto out_free_unlock; - } - - if (gc->request) { - /* gc->request may sleep */ - spin_unlock_irqrestore(&gpio_lock, flags); + if (guard.gc->request) { offset = gpio_chip_hwgpio(desc); - if (gpiochip_line_is_valid(gc, offset)) - ret = gc->request(gc, offset); + if (gpiochip_line_is_valid(guard.gc, offset)) + ret = guard.gc->request(guard.gc, offset); else ret = -EINVAL; - spin_lock_irqsave(&gpio_lock, flags); + if (ret) + goto out_clear_bit; + } - if (ret) { - desc_set_label(desc, NULL); - clear_bit(FLAG_REQUESTED, &desc->flags); - goto out_free_unlock; - } - } - if (gc->get_direction) { - /* gc->get_direction may sleep */ - spin_unlock_irqrestore(&gpio_lock, flags); + if (guard.gc->get_direction) gpiod_get_direction(desc); - spin_lock_irqsave(&gpio_lock, flags); - } - spin_unlock_irqrestore(&gpio_lock, flags); + + ret = desc_set_label(desc, label ? : "?"); + if (ret) + goto out_clear_bit; + return 0; -out_free_unlock: - spin_unlock_irqrestore(&gpio_lock, flags); - kfree_const(label); +out_clear_bit: + clear_bit(FLAG_REQUESTED, &desc->flags); return ret; } @@ -2254,19 +2279,12 @@ static int validate_desc(const struct gpio_desc *desc, const char *func) { if (!desc) return 0; + if (IS_ERR(desc)) { pr_warn("%s: invalid GPIO (errorpointer)\n", func); return PTR_ERR(desc); } - if (!desc->gdev) { - pr_warn("%s: invalid GPIO (no device)\n", func); - return -EINVAL; - } - if (!desc->gdev->chip) { - dev_warn(&desc->gdev->dev, - "%s: backing chip is gone\n", func); - return 0; - } + return 1; } @@ -2302,60 +2320,45 @@ int gpiod_request(struct gpio_desc *desc, const char *label) return ret; } -static bool gpiod_free_commit(struct gpio_desc *desc) +static void gpiod_free_commit(struct gpio_desc *desc) { - struct gpio_chip *gc; unsigned long flags; - bool ret = false; might_sleep(); - spin_lock_irqsave(&gpio_lock, flags); + CLASS(gpio_chip_guard, guard)(desc); - gc = desc->gdev->chip; - if (gc && test_bit(FLAG_REQUESTED, &desc->flags)) { - if (gc->free) { - spin_unlock_irqrestore(&gpio_lock, flags); - might_sleep_if(gc->can_sleep); - gc->free(gc, gpio_chip_hwgpio(desc)); - spin_lock_irqsave(&gpio_lock, flags); - } - kfree_const(desc->label); - desc_set_label(desc, NULL); - clear_bit(FLAG_ACTIVE_LOW, &desc->flags); - clear_bit(FLAG_REQUESTED, &desc->flags); - clear_bit(FLAG_OPEN_DRAIN, &desc->flags); - clear_bit(FLAG_OPEN_SOURCE, &desc->flags); - clear_bit(FLAG_PULL_UP, &desc->flags); - clear_bit(FLAG_PULL_DOWN, &desc->flags); - clear_bit(FLAG_BIAS_DISABLE, &desc->flags); - clear_bit(FLAG_EDGE_RISING, &desc->flags); - clear_bit(FLAG_EDGE_FALLING, &desc->flags); - clear_bit(FLAG_IS_HOGGED, &desc->flags); + flags = READ_ONCE(desc->flags); + + if (guard.gc && test_bit(FLAG_REQUESTED, &flags)) { + if (guard.gc->free) + guard.gc->free(guard.gc, gpio_chip_hwgpio(desc)); + + clear_bit(FLAG_ACTIVE_LOW, &flags); + clear_bit(FLAG_REQUESTED, &flags); + clear_bit(FLAG_OPEN_DRAIN, &flags); + clear_bit(FLAG_OPEN_SOURCE, &flags); + clear_bit(FLAG_PULL_UP, &flags); + clear_bit(FLAG_PULL_DOWN, &flags); + clear_bit(FLAG_BIAS_DISABLE, &flags); + clear_bit(FLAG_EDGE_RISING, &flags); + clear_bit(FLAG_EDGE_FALLING, &flags); + clear_bit(FLAG_IS_HOGGED, &flags); #ifdef CONFIG_OF_DYNAMIC - desc->hog = NULL; + WRITE_ONCE(desc->hog, NULL); #endif - ret = true; + desc_set_label(desc, NULL); + WRITE_ONCE(desc->flags, flags); + + gpiod_line_state_notify(desc, GPIOLINE_CHANGED_RELEASED); } - - spin_unlock_irqrestore(&gpio_lock, flags); - gpiod_line_state_notify(desc, GPIOLINE_CHANGED_RELEASED); - - return ret; } void gpiod_free(struct gpio_desc *desc) { - /* - * We must not use VALIDATE_DESC_VOID() as the underlying gdev->chip - * may already be NULL but we still want to put the references. - */ - if (!desc) - return; - - if (!gpiod_free_commit(desc)) - WARN_ON(1); + VALIDATE_DESC_VOID(desc); + gpiod_free_commit(desc); module_put(desc->gdev->owner); gpio_device_put(desc->gdev); } @@ -2381,20 +2384,12 @@ char *gpiochip_dup_line_label(struct gpio_chip *gc, unsigned int offset) if (IS_ERR(desc)) return NULL; - guard(spinlock_irqsave)(&gpio_lock); - if (!test_bit(FLAG_REQUESTED, &desc->flags)) return NULL; - /* - * FIXME: Once we mark gpiod_direction_input/output() and - * gpiod_get_direction() with might_sleep(), we'll be able to protect - * the GPIO descriptors with mutex (while value setting operations will - * become lockless). - * - * Until this happens, this allocation needs to be atomic. - */ - label = kstrdup(desc->label, GFP_ATOMIC); + guard(srcu)(&desc->srcu); + + label = kstrdup(gpiod_get_label(desc), GFP_KERNEL); if (!label) return ERR_PTR(-ENOMEM); @@ -2489,11 +2484,14 @@ static int gpio_set_config_with_argument(struct gpio_desc *desc, enum pin_config_param mode, u32 argument) { - struct gpio_chip *gc = desc->gdev->chip; unsigned long config; + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + config = pinconf_to_config_packed(mode, argument); - return gpio_do_set_config(gc, gpio_chip_hwgpio(desc), config); + return gpio_do_set_config(guard.gc, gpio_chip_hwgpio(desc), config); } static int gpio_set_config_with_argument_optional(struct gpio_desc *desc, @@ -2527,13 +2525,16 @@ static int gpio_set_config(struct gpio_desc *desc, enum pin_config_param mode) static int gpio_set_bias(struct gpio_desc *desc) { enum pin_config_param bias; + unsigned long flags; unsigned int arg; - if (test_bit(FLAG_BIAS_DISABLE, &desc->flags)) + flags = READ_ONCE(desc->flags); + + if (test_bit(FLAG_BIAS_DISABLE, &flags)) bias = PIN_CONFIG_BIAS_DISABLE; - else if (test_bit(FLAG_PULL_UP, &desc->flags)) + else if (test_bit(FLAG_PULL_UP, &flags)) bias = PIN_CONFIG_BIAS_PULL_UP; - else if (test_bit(FLAG_PULL_DOWN, &desc->flags)) + else if (test_bit(FLAG_PULL_DOWN, &flags)) bias = PIN_CONFIG_BIAS_PULL_DOWN; else return 0; @@ -2580,18 +2581,20 @@ int gpio_set_debounce_timeout(struct gpio_desc *desc, unsigned int debounce) */ int gpiod_direction_input(struct gpio_desc *desc) { - struct gpio_chip *gc; int ret = 0; VALIDATE_DESC(desc); - gc = desc->gdev->chip; + + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; /* * It is legal to have no .get() and .direction_input() specified if * the chip is output-only, but you can't specify .direction_input() * and not support the .get() operation, that doesn't make sense. */ - if (!gc->get && gc->direction_input) { + if (!guard.gc->get && guard.gc->direction_input) { gpiod_warn(desc, "%s: missing get() but have direction_input()\n", __func__); @@ -2604,10 +2607,12 @@ int gpiod_direction_input(struct gpio_desc *desc) * direction (if .get_direction() is supported) else we silently * assume we are in input mode after this. */ - if (gc->direction_input) { - ret = gc->direction_input(gc, gpio_chip_hwgpio(desc)); - } else if (gc->get_direction && - (gc->get_direction(gc, gpio_chip_hwgpio(desc)) != 1)) { + if (guard.gc->direction_input) { + ret = guard.gc->direction_input(guard.gc, + gpio_chip_hwgpio(desc)); + } else if (guard.gc->get_direction && + (guard.gc->get_direction(guard.gc, + gpio_chip_hwgpio(desc)) != 1)) { gpiod_warn(desc, "%s: missing direction_input() operation and line is output\n", __func__); @@ -2626,28 +2631,31 @@ EXPORT_SYMBOL_GPL(gpiod_direction_input); static int gpiod_direction_output_raw_commit(struct gpio_desc *desc, int value) { - struct gpio_chip *gc = desc->gdev->chip; - int val = !!value; - int ret = 0; + int val = !!value, ret = 0; + + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; /* * It's OK not to specify .direction_output() if the gpiochip is * output-only, but if there is then not even a .set() operation it * is pretty tricky to drive the output line. */ - if (!gc->set && !gc->direction_output) { + if (!guard.gc->set && !guard.gc->direction_output) { gpiod_warn(desc, "%s: missing set() and direction_output() operations\n", __func__); return -EIO; } - if (gc->direction_output) { - ret = gc->direction_output(gc, gpio_chip_hwgpio(desc), val); + if (guard.gc->direction_output) { + ret = guard.gc->direction_output(guard.gc, + gpio_chip_hwgpio(desc), val); } else { /* Check that we are in output mode if we can */ - if (gc->get_direction && - gc->get_direction(gc, gpio_chip_hwgpio(desc))) { + if (guard.gc->get_direction && + guard.gc->get_direction(guard.gc, gpio_chip_hwgpio(desc))) { gpiod_warn(desc, "%s: missing direction_output() operation\n", __func__); @@ -2657,7 +2665,7 @@ static int gpiod_direction_output_raw_commit(struct gpio_desc *desc, int value) * If we can't actively set the direction, we are some * output-only chip, so just drive the output as desired. */ - gc->set(gc, gpio_chip_hwgpio(desc), val); + guard.gc->set(guard.gc, gpio_chip_hwgpio(desc), val); } if (!ret) @@ -2699,24 +2707,28 @@ EXPORT_SYMBOL_GPL(gpiod_direction_output_raw); */ int gpiod_direction_output(struct gpio_desc *desc, int value) { + unsigned long flags; int ret; VALIDATE_DESC(desc); - if (test_bit(FLAG_ACTIVE_LOW, &desc->flags)) + + flags = READ_ONCE(desc->flags); + + if (test_bit(FLAG_ACTIVE_LOW, &flags)) value = !value; else value = !!value; /* GPIOs used for enabled IRQs shall not be set as output */ - if (test_bit(FLAG_USED_AS_IRQ, &desc->flags) && - test_bit(FLAG_IRQ_IS_ENABLED, &desc->flags)) { + if (test_bit(FLAG_USED_AS_IRQ, &flags) && + test_bit(FLAG_IRQ_IS_ENABLED, &flags)) { gpiod_err(desc, "%s: tried to set a GPIO tied to an IRQ as output\n", __func__); return -EIO; } - if (test_bit(FLAG_OPEN_DRAIN, &desc->flags)) { + if (test_bit(FLAG_OPEN_DRAIN, &flags)) { /* First see if we can enable open drain in hardware */ ret = gpio_set_config(desc, PIN_CONFIG_DRIVE_OPEN_DRAIN); if (!ret) @@ -2726,7 +2738,7 @@ int gpiod_direction_output(struct gpio_desc *desc, int value) ret = gpiod_direction_input(desc); goto set_output_flag; } - } else if (test_bit(FLAG_OPEN_SOURCE, &desc->flags)) { + } else if (test_bit(FLAG_OPEN_SOURCE, &flags)) { ret = gpio_set_config(desc, PIN_CONFIG_DRIVE_OPEN_SOURCE); if (!ret) goto set_output_value; @@ -2769,17 +2781,20 @@ EXPORT_SYMBOL_GPL(gpiod_direction_output); int gpiod_enable_hw_timestamp_ns(struct gpio_desc *desc, unsigned long flags) { int ret = 0; - struct gpio_chip *gc; VALIDATE_DESC(desc); - gc = desc->gdev->chip; - if (!gc->en_hw_timestamp) { + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + + if (!guard.gc->en_hw_timestamp) { gpiod_warn(desc, "%s: hw ts not supported\n", __func__); return -ENOTSUPP; } - ret = gc->en_hw_timestamp(gc, gpio_chip_hwgpio(desc), flags); + ret = guard.gc->en_hw_timestamp(guard.gc, + gpio_chip_hwgpio(desc), flags); if (ret) gpiod_warn(desc, "%s: hw ts request failed\n", __func__); @@ -2798,17 +2813,20 @@ EXPORT_SYMBOL_GPL(gpiod_enable_hw_timestamp_ns); int gpiod_disable_hw_timestamp_ns(struct gpio_desc *desc, unsigned long flags) { int ret = 0; - struct gpio_chip *gc; VALIDATE_DESC(desc); - gc = desc->gdev->chip; - if (!gc->dis_hw_timestamp) { + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + + if (!guard.gc->dis_hw_timestamp) { gpiod_warn(desc, "%s: hw ts not supported\n", __func__); return -ENOTSUPP; } - ret = gc->dis_hw_timestamp(gc, gpio_chip_hwgpio(desc), flags); + ret = guard.gc->dis_hw_timestamp(guard.gc, gpio_chip_hwgpio(desc), + flags); if (ret) gpiod_warn(desc, "%s: hw ts release failed\n", __func__); @@ -2827,12 +2845,13 @@ EXPORT_SYMBOL_GPL(gpiod_disable_hw_timestamp_ns); */ int gpiod_set_config(struct gpio_desc *desc, unsigned long config) { - struct gpio_chip *gc; - VALIDATE_DESC(desc); - gc = desc->gdev->chip; - return gpio_do_set_config(gc, gpio_chip_hwgpio(desc), config); + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + + return gpio_do_set_config(guard.gc, gpio_chip_hwgpio(desc), config); } EXPORT_SYMBOL_GPL(gpiod_set_config); @@ -2930,10 +2949,19 @@ static int gpio_chip_get_value(struct gpio_chip *gc, const struct gpio_desc *des static int gpiod_get_raw_value_commit(const struct gpio_desc *desc) { + struct gpio_device *gdev; struct gpio_chip *gc; int value; - gc = desc->gdev->chip; + /* FIXME Unable to use gpio_chip_guard due to const desc. */ + gdev = desc->gdev; + + guard(srcu)(&gdev->srcu); + + gc = srcu_dereference(gdev->chip, &gdev->srcu); + if (!gc) + return -ENODEV; + value = gpio_chip_get_value(gc, desc); value = value < 0 ? value : !!value; trace_gpio_value(desc_to_gpio(desc), 1, value); @@ -2959,6 +2987,14 @@ static int gpio_chip_get_multiple(struct gpio_chip *gc, return -EIO; } +/* The 'other' chip must be protected with its GPIO device's SRCU. */ +static bool gpio_device_chip_cmp(struct gpio_device *gdev, struct gpio_chip *gc) +{ + guard(srcu)(&gdev->srcu); + + return gc == srcu_dereference(gdev->chip, &gdev->srcu); +} + int gpiod_get_array_value_complex(bool raw, bool can_sleep, unsigned int array_size, struct gpio_desc **desc_array, @@ -2996,33 +3032,36 @@ int gpiod_get_array_value_complex(bool raw, bool can_sleep, } while (i < array_size) { - struct gpio_chip *gc = desc_array[i]->gdev->chip; DECLARE_BITMAP(fastpath_mask, FASTPATH_NGPIO); DECLARE_BITMAP(fastpath_bits, FASTPATH_NGPIO); unsigned long *mask, *bits; int first, j; - if (likely(gc->ngpio <= FASTPATH_NGPIO)) { + CLASS(gpio_chip_guard, guard)(desc_array[i]); + if (!guard.gc) + return -ENODEV; + + if (likely(guard.gc->ngpio <= FASTPATH_NGPIO)) { mask = fastpath_mask; bits = fastpath_bits; } else { gfp_t flags = can_sleep ? GFP_KERNEL : GFP_ATOMIC; - mask = bitmap_alloc(gc->ngpio, flags); + mask = bitmap_alloc(guard.gc->ngpio, flags); if (!mask) return -ENOMEM; - bits = bitmap_alloc(gc->ngpio, flags); + bits = bitmap_alloc(guard.gc->ngpio, flags); if (!bits) { bitmap_free(mask); return -ENOMEM; } } - bitmap_zero(mask, gc->ngpio); + bitmap_zero(mask, guard.gc->ngpio); if (!can_sleep) - WARN_ON(gc->can_sleep); + WARN_ON(guard.gc->can_sleep); /* collect all inputs belonging to the same chip */ first = i; @@ -3037,9 +3076,9 @@ int gpiod_get_array_value_complex(bool raw, bool can_sleep, i = find_next_zero_bit(array_info->get_mask, array_size, i); } while ((i < array_size) && - (desc_array[i]->gdev->chip == gc)); + gpio_device_chip_cmp(desc_array[i]->gdev, guard.gc)); - ret = gpio_chip_get_multiple(gc, mask, bits); + ret = gpio_chip_get_multiple(guard.gc, mask, bits); if (ret) { if (mask != fastpath_mask) bitmap_free(mask); @@ -3086,7 +3125,7 @@ int gpiod_get_raw_value(const struct gpio_desc *desc) { VALIDATE_DESC(desc); /* Should be using gpiod_get_raw_value_cansleep() */ - WARN_ON(desc->gdev->chip->can_sleep); + WARN_ON(desc->gdev->can_sleep); return gpiod_get_raw_value_commit(desc); } EXPORT_SYMBOL_GPL(gpiod_get_raw_value); @@ -3107,7 +3146,7 @@ int gpiod_get_value(const struct gpio_desc *desc) VALIDATE_DESC(desc); /* Should be using gpiod_get_value_cansleep() */ - WARN_ON(desc->gdev->chip->can_sleep); + WARN_ON(desc->gdev->can_sleep); value = gpiod_get_raw_value_commit(desc); if (value < 0) @@ -3180,14 +3219,16 @@ EXPORT_SYMBOL_GPL(gpiod_get_array_value); */ static void gpio_set_open_drain_value_commit(struct gpio_desc *desc, bool value) { - int ret = 0; - struct gpio_chip *gc = desc->gdev->chip; - int offset = gpio_chip_hwgpio(desc); + int ret = 0, offset = gpio_chip_hwgpio(desc); + + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return; if (value) { - ret = gc->direction_input(gc, offset); + ret = guard.gc->direction_input(guard.gc, offset); } else { - ret = gc->direction_output(gc, offset, 0); + ret = guard.gc->direction_output(guard.gc, offset, 0); if (!ret) set_bit(FLAG_IS_OUT, &desc->flags); } @@ -3205,16 +3246,18 @@ static void gpio_set_open_drain_value_commit(struct gpio_desc *desc, bool value) */ static void gpio_set_open_source_value_commit(struct gpio_desc *desc, bool value) { - int ret = 0; - struct gpio_chip *gc = desc->gdev->chip; - int offset = gpio_chip_hwgpio(desc); + int ret = 0, offset = gpio_chip_hwgpio(desc); + + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return; if (value) { - ret = gc->direction_output(gc, offset, 1); + ret = guard.gc->direction_output(guard.gc, offset, 1); if (!ret) set_bit(FLAG_IS_OUT, &desc->flags); } else { - ret = gc->direction_input(gc, offset); + ret = guard.gc->direction_input(guard.gc, offset); } trace_gpio_direction(desc_to_gpio(desc), !value, ret); if (ret < 0) @@ -3225,11 +3268,12 @@ static void gpio_set_open_source_value_commit(struct gpio_desc *desc, bool value static void gpiod_set_raw_value_commit(struct gpio_desc *desc, bool value) { - struct gpio_chip *gc; + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return; - gc = desc->gdev->chip; trace_gpio_value(desc_to_gpio(desc), 0, value); - gc->set(gc, gpio_chip_hwgpio(desc), value); + guard.gc->set(guard.gc, gpio_chip_hwgpio(desc), value); } /* @@ -3290,33 +3334,36 @@ int gpiod_set_array_value_complex(bool raw, bool can_sleep, } while (i < array_size) { - struct gpio_chip *gc = desc_array[i]->gdev->chip; DECLARE_BITMAP(fastpath_mask, FASTPATH_NGPIO); DECLARE_BITMAP(fastpath_bits, FASTPATH_NGPIO); unsigned long *mask, *bits; int count = 0; - if (likely(gc->ngpio <= FASTPATH_NGPIO)) { + CLASS(gpio_chip_guard, guard)(desc_array[i]); + if (!guard.gc) + return -ENODEV; + + if (likely(guard.gc->ngpio <= FASTPATH_NGPIO)) { mask = fastpath_mask; bits = fastpath_bits; } else { gfp_t flags = can_sleep ? GFP_KERNEL : GFP_ATOMIC; - mask = bitmap_alloc(gc->ngpio, flags); + mask = bitmap_alloc(guard.gc->ngpio, flags); if (!mask) return -ENOMEM; - bits = bitmap_alloc(gc->ngpio, flags); + bits = bitmap_alloc(guard.gc->ngpio, flags); if (!bits) { bitmap_free(mask); return -ENOMEM; } } - bitmap_zero(mask, gc->ngpio); + bitmap_zero(mask, guard.gc->ngpio); if (!can_sleep) - WARN_ON(gc->can_sleep); + WARN_ON(guard.gc->can_sleep); do { struct gpio_desc *desc = desc_array[i]; @@ -3352,10 +3399,10 @@ int gpiod_set_array_value_complex(bool raw, bool can_sleep, i = find_next_zero_bit(array_info->set_mask, array_size, i); } while ((i < array_size) && - (desc_array[i]->gdev->chip == gc)); + gpio_device_chip_cmp(desc_array[i]->gdev, guard.gc)); /* push collected bits to outputs */ if (count != 0) - gpio_chip_set_multiple(gc, mask, bits); + gpio_chip_set_multiple(guard.gc, mask, bits); if (mask != fastpath_mask) bitmap_free(mask); @@ -3380,7 +3427,7 @@ void gpiod_set_raw_value(struct gpio_desc *desc, int value) { VALIDATE_DESC_VOID(desc); /* Should be using gpiod_set_raw_value_cansleep() */ - WARN_ON(desc->gdev->chip->can_sleep); + WARN_ON(desc->gdev->can_sleep); gpiod_set_raw_value_commit(desc, value); } EXPORT_SYMBOL_GPL(gpiod_set_raw_value); @@ -3421,7 +3468,7 @@ void gpiod_set_value(struct gpio_desc *desc, int value) { VALIDATE_DESC_VOID(desc); /* Should be using gpiod_set_value_cansleep() */ - WARN_ON(desc->gdev->chip->can_sleep); + WARN_ON(desc->gdev->can_sleep); gpiod_set_value_nocheck(desc, value); } EXPORT_SYMBOL_GPL(gpiod_set_value); @@ -3485,7 +3532,7 @@ EXPORT_SYMBOL_GPL(gpiod_set_array_value); int gpiod_cansleep(const struct gpio_desc *desc) { VALIDATE_DESC(desc); - return desc->gdev->chip->can_sleep; + return desc->gdev->can_sleep; } EXPORT_SYMBOL_GPL(gpiod_cansleep); @@ -3497,16 +3544,8 @@ EXPORT_SYMBOL_GPL(gpiod_cansleep); int gpiod_set_consumer_name(struct gpio_desc *desc, const char *name) { VALIDATE_DESC(desc); - if (name) { - name = kstrdup_const(name, GFP_KERNEL); - if (!name) - return -ENOMEM; - } - kfree_const(desc->label); - desc_set_label(desc, name); - - return 0; + return desc_set_label(desc, name); } EXPORT_SYMBOL_GPL(gpiod_set_consumer_name); @@ -3519,6 +3558,7 @@ EXPORT_SYMBOL_GPL(gpiod_set_consumer_name); */ int gpiod_to_irq(const struct gpio_desc *desc) { + struct gpio_device *gdev; struct gpio_chip *gc; int offset; @@ -3527,10 +3567,16 @@ int gpiod_to_irq(const struct gpio_desc *desc) * requires this function to not return zero on an invalid descriptor * but rather a negative error number. */ - if (!desc || IS_ERR(desc) || !desc->gdev || !desc->gdev->chip) + if (!desc || IS_ERR(desc)) return -EINVAL; - gc = desc->gdev->chip; + gdev = desc->gdev; + /* FIXME Cannot use gpio_chip_guard due to const desc. */ + guard(srcu)(&gdev->srcu); + gc = srcu_dereference(gdev->chip, &gdev->srcu); + if (!gc) + return -ENODEV; + offset = gpio_chip_hwgpio(desc); if (gc->to_irq) { int retirq = gc->to_irq(gc, offset); @@ -3597,14 +3643,6 @@ int gpiochip_lock_as_irq(struct gpio_chip *gc, unsigned int offset) set_bit(FLAG_USED_AS_IRQ, &desc->flags); set_bit(FLAG_IRQ_IS_ENABLED, &desc->flags); - /* - * If the consumer has not set up a label (such as when the - * IRQ is referenced from .to_irq()) we set up a label here - * so it is clear this is used as an interrupt. - */ - if (!desc->label) - desc_set_label(desc, "interrupt"); - return 0; } EXPORT_SYMBOL_GPL(gpiochip_lock_as_irq); @@ -3627,10 +3665,6 @@ void gpiochip_unlock_as_irq(struct gpio_chip *gc, unsigned int offset) clear_bit(FLAG_USED_AS_IRQ, &desc->flags); clear_bit(FLAG_IRQ_IS_ENABLED, &desc->flags); - - /* If we only had this marking, erase it */ - if (desc->label && !strcmp(desc->label, "interrupt")) - desc_set_label(desc, NULL); } EXPORT_SYMBOL_GPL(gpiochip_unlock_as_irq); @@ -4138,39 +4172,48 @@ static struct gpio_desc *gpiod_find_by_fwnode(struct fwnode_handle *fwnode, return desc; } -static struct gpio_desc *gpiod_find_and_request(struct device *consumer, - struct fwnode_handle *fwnode, - const char *con_id, - unsigned int idx, - enum gpiod_flags flags, - const char *label, - bool platform_lookup_allowed) +struct gpio_desc *gpiod_find_and_request(struct device *consumer, + struct fwnode_handle *fwnode, + const char *con_id, + unsigned int idx, + enum gpiod_flags flags, + const char *label, + bool platform_lookup_allowed) { unsigned long lookupflags = GPIO_LOOKUP_FLAGS_DEFAULT; - struct gpio_desc *desc; - int ret; - - desc = gpiod_find_by_fwnode(fwnode, consumer, con_id, idx, &flags, &lookupflags); - if (gpiod_not_found(desc) && platform_lookup_allowed) { - /* - * Either we are not using DT or ACPI, or their lookup did not - * return a result. In that case, use platform lookup as a - * fallback. - */ - dev_dbg(consumer, "using lookup tables for GPIO lookup\n"); - desc = gpiod_find(consumer, con_id, idx, &lookupflags); - } - - if (IS_ERR(desc)) { - dev_dbg(consumer, "No GPIO consumer %s found\n", con_id); - return desc; - } - /* - * If a connection label was passed use that, else attempt to use - * the device name as label + * scoped_guard() is implemented as a for loop, meaning static + * analyzers will complain about these two not being initialized. */ - ret = gpiod_request(desc, label); + struct gpio_desc *desc = NULL; + int ret = 0; + + scoped_guard(srcu, &gpio_devices_srcu) { + desc = gpiod_find_by_fwnode(fwnode, consumer, con_id, idx, + &flags, &lookupflags); + if (gpiod_not_found(desc) && platform_lookup_allowed) { + /* + * Either we are not using DT or ACPI, or their lookup + * did not return a result. In that case, use platform + * lookup as a fallback. + */ + dev_dbg(consumer, + "using lookup tables for GPIO lookup\n"); + desc = gpiod_find(consumer, con_id, idx, &lookupflags); + } + + if (IS_ERR(desc)) { + dev_dbg(consumer, "No GPIO consumer %s found\n", + con_id); + return desc; + } + + /* + * If a connection label was passed use that, else attempt to use + * the device name as label + */ + ret = gpiod_request(desc, label); + } if (ret) { if (!(ret == -EBUSY && flags & GPIOD_FLAGS_BIT_NONEXCLUSIVE)) return ERR_PTR(ret); @@ -4243,9 +4286,9 @@ int gpiod_count(struct device *dev, const char *con_id) int count = -ENOENT; if (is_of_node(fwnode)) - count = of_gpio_get_count(dev, con_id); + count = of_gpio_count(fwnode, con_id); else if (is_acpi_node(fwnode)) - count = acpi_gpio_count(dev, con_id); + count = acpi_gpio_count(fwnode, con_id); else if (is_software_node(fwnode)) count = swnode_gpio_count(fwnode, con_id); @@ -4429,26 +4472,30 @@ EXPORT_SYMBOL_GPL(gpiod_get_index_optional); int gpiod_hog(struct gpio_desc *desc, const char *name, unsigned long lflags, enum gpiod_flags dflags) { - struct gpio_chip *gc; + struct gpio_device *gdev = desc->gdev; struct gpio_desc *local_desc; int hwnum; int ret; - gc = gpiod_to_chip(desc); + CLASS(gpio_chip_guard, guard)(desc); + if (!guard.gc) + return -ENODEV; + + if (test_and_set_bit(FLAG_IS_HOGGED, &desc->flags)) + return 0; + hwnum = gpio_chip_hwgpio(desc); - local_desc = gpiochip_request_own_desc(gc, hwnum, name, + local_desc = gpiochip_request_own_desc(guard.gc, hwnum, name, lflags, dflags); if (IS_ERR(local_desc)) { + clear_bit(FLAG_IS_HOGGED, &desc->flags); ret = PTR_ERR(local_desc); pr_err("requesting hog GPIO %s (chip %s, offset %d) failed, %d\n", - name, gc->label, hwnum, ret); + name, gdev->label, hwnum, ret); return ret; } - /* Mark GPIO as hogged so it can be identified and removed later */ - set_bit(FLAG_IS_HOGGED, &desc->flags); - gpiod_dbg(desc, "hogged as %s%s\n", (dflags & GPIOD_FLAGS_BIT_DIR_OUT) ? "output" : "input", (dflags & GPIOD_FLAGS_BIT_DIR_OUT) ? @@ -4712,13 +4759,22 @@ core_initcall(gpiolib_dev_init); static void gpiolib_dbg_show(struct seq_file *s, struct gpio_device *gdev) { - struct gpio_chip *gc = gdev->chip; bool active_low, is_irq, is_out; unsigned int gpio = gdev->base; struct gpio_desc *desc; + struct gpio_chip *gc; int value; + guard(srcu)(&gdev->srcu); + + gc = srcu_dereference(gdev->chip, &gdev->srcu); + if (!gc) { + seq_puts(s, "Underlying GPIO chip is gone\n"); + return; + } + for_each_gpio_desc(gc, desc) { + guard(srcu)(&desc->srcu); if (test_bit(FLAG_REQUESTED, &desc->flags)) { gpiod_get_direction(desc); is_out = test_bit(FLAG_IS_OUT, &desc->flags); @@ -4726,7 +4782,7 @@ static void gpiolib_dbg_show(struct seq_file *s, struct gpio_device *gdev) is_irq = test_bit(FLAG_USED_AS_IRQ, &desc->flags); active_low = test_bit(FLAG_ACTIVE_LOW, &desc->flags); seq_printf(s, " gpio-%-3d (%-20.20s|%-20.20s) %s %s %s%s\n", - gpio, desc->name ?: "", desc->label, + gpio, desc->name ?: "", gpiod_get_label(desc), is_out ? "out" : "in ", value >= 0 ? (value ? "hi" : "lo") : "? ", is_irq ? "IRQ " : "", @@ -4739,61 +4795,72 @@ static void gpiolib_dbg_show(struct seq_file *s, struct gpio_device *gdev) } } +struct gpiolib_seq_priv { + bool newline; + int idx; +}; + static void *gpiolib_seq_start(struct seq_file *s, loff_t *pos) { - unsigned long flags; - struct gpio_device *gdev = NULL; + struct gpiolib_seq_priv *priv; + struct gpio_device *gdev; loff_t index = *pos; - s->private = ""; + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return NULL; - spin_lock_irqsave(&gpio_lock, flags); - list_for_each_entry(gdev, &gpio_devices, list) - if (index-- == 0) { - spin_unlock_irqrestore(&gpio_lock, flags); + s->private = priv; + priv->idx = srcu_read_lock(&gpio_devices_srcu); + + list_for_each_entry_srcu(gdev, &gpio_devices, list, + srcu_read_lock_held(&gpio_devices_srcu)) { + if (index-- == 0) return gdev; - } - spin_unlock_irqrestore(&gpio_lock, flags); + } return NULL; } static void *gpiolib_seq_next(struct seq_file *s, void *v, loff_t *pos) { - unsigned long flags; - struct gpio_device *gdev = v; - void *ret = NULL; + struct gpiolib_seq_priv *priv = s->private; + struct gpio_device *gdev = v, *next; - spin_lock_irqsave(&gpio_lock, flags); - if (list_is_last(&gdev->list, &gpio_devices)) - ret = NULL; - else - ret = list_first_entry(&gdev->list, struct gpio_device, list); - spin_unlock_irqrestore(&gpio_lock, flags); - - s->private = "\n"; + next = list_entry_rcu(gdev->list.next, struct gpio_device, list); + gdev = &next->list == &gpio_devices ? NULL : next; + priv->newline = true; ++*pos; - return ret; + return gdev; } static void gpiolib_seq_stop(struct seq_file *s, void *v) { + struct gpiolib_seq_priv *priv = s->private; + + srcu_read_unlock(&gpio_devices_srcu, priv->idx); + kfree(priv); } static int gpiolib_seq_show(struct seq_file *s, void *v) { + struct gpiolib_seq_priv *priv = s->private; struct gpio_device *gdev = v; - struct gpio_chip *gc = gdev->chip; + struct gpio_chip *gc; struct device *parent; + guard(srcu)(&gdev->srcu); + + gc = srcu_dereference(gdev->chip, &gdev->srcu); if (!gc) { - seq_printf(s, "%s%s: (dangling chip)", (char *)s->private, + seq_printf(s, "%s%s: (dangling chip)", + priv->newline ? "\n" : "", dev_name(&gdev->dev)); return 0; } - seq_printf(s, "%s%s: GPIOs %d-%d", (char *)s->private, + seq_printf(s, "%s%s: GPIOs %d-%d", priv->newline ? "\n" : "", dev_name(&gdev->dev), gdev->base, gdev->base + gdev->ngpio - 1); parent = gc->parent; diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index a4a2520b5f31..f67d5991ab1c 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -16,7 +16,7 @@ #include #include #include -#include +#include #define GPIOCHIP_NAME "gpiochip" @@ -33,6 +33,8 @@ * @descs: array of ngpio descriptors. * @ngpio: the number of GPIO lines on this GPIO device, equal to the size * of the @descs array. + * @can_sleep: indicate whether the GPIO chip driver's callbacks can sleep + * implying that they cannot be used from atomic context * @base: GPIO base in the DEPRECATED global Linux GPIO numberspace, assigned * at device creation time. * @label: a descriptive name for the GPIO device, such as the part number @@ -43,9 +45,7 @@ * requested, released or reconfigured * @device_notifier: used to notify character device wait queues about the GPIO * device being unregistered - * @sem: protects the structure from a NULL-pointer dereference of @chip by - * user-space operations when the device gets unregistered during - * a hot-unplug event + * @srcu: protects the pointer to the underlying GPIO chip * @pin_ranges: range of pins served by the GPIO driver * * This state container holds most of the runtime variable data @@ -59,16 +59,17 @@ struct gpio_device { int id; struct device *mockdev; struct module *owner; - struct gpio_chip *chip; + struct gpio_chip __rcu *chip; struct gpio_desc *descs; int base; u16 ngpio; + bool can_sleep; const char *label; void *data; struct list_head list; struct blocking_notifier_head line_state_notifier; struct blocking_notifier_head device_notifier; - struct rw_semaphore sem; + struct srcu_struct srcu; #ifdef CONFIG_PINCTRL /* @@ -134,9 +135,6 @@ int gpiod_set_array_value_complex(bool raw, bool can_sleep, int gpiod_set_transitory(struct gpio_desc *desc, bool transitory); -extern spinlock_t gpio_lock; -extern struct list_head gpio_devices; - void gpiod_line_state_notify(struct gpio_desc *desc, unsigned long action); /** @@ -147,6 +145,7 @@ void gpiod_line_state_notify(struct gpio_desc *desc, unsigned long action); * @label: Name of the consumer * @name: Line name * @hog: Pointer to the device node that hogs this line (if any) + * @srcu: SRCU struct protecting the label pointer. * * These are obtained using gpiod_get() and are preferable to the old * integer-based handles. @@ -178,16 +177,38 @@ struct gpio_desc { #define FLAG_EVENT_CLOCK_HTE 19 /* GPIO CDEV reports hardware timestamps in events */ /* Connection label */ - const char *label; + const char __rcu *label; /* Name of the GPIO */ const char *name; #ifdef CONFIG_OF_DYNAMIC struct device_node *hog; #endif + struct srcu_struct srcu; }; #define gpiod_not_found(desc) (IS_ERR(desc) && PTR_ERR(desc) == -ENOENT) +struct gpio_chip_guard { + struct gpio_device *gdev; + struct gpio_chip *gc; + int idx; +}; + +DEFINE_CLASS(gpio_chip_guard, + struct gpio_chip_guard, + srcu_read_unlock(&_T.gdev->srcu, _T.idx), + ({ + struct gpio_chip_guard _guard; + + _guard.gdev = desc->gdev; + _guard.idx = srcu_read_lock(&_guard.gdev->srcu); + _guard.gc = srcu_dereference(_guard.gdev->chip, + &_guard.gdev->srcu); + + _guard; + }), + struct gpio_desc *desc) + int gpiod_request(struct gpio_desc *desc, const char *label); void gpiod_free(struct gpio_desc *desc); @@ -202,12 +223,21 @@ static inline int gpiod_request_user(struct gpio_desc *desc, const char *label) return ret; } +struct gpio_desc *gpiod_find_and_request(struct device *consumer, + struct fwnode_handle *fwnode, + const char *con_id, + unsigned int idx, + enum gpiod_flags flags, + const char *label, + bool platform_lookup_allowed); + int gpiod_configure_flags(struct gpio_desc *desc, const char *con_id, unsigned long lflags, enum gpiod_flags dflags); int gpio_set_debounce_timeout(struct gpio_desc *desc, unsigned int debounce); int gpiod_hog(struct gpio_desc *desc, const char *name, unsigned long lflags, enum gpiod_flags dflags); int gpiochip_get_ngpios(struct gpio_chip *gc, struct device *dev); +const char *gpiod_get_label(struct gpio_desc *desc); /* * Return the GPIO number of the passed descriptor relative to its chip @@ -219,31 +249,32 @@ static inline int gpio_chip_hwgpio(const struct gpio_desc *desc) /* With descriptor prefix */ -#define gpiod_emerg(desc, fmt, ...) \ - pr_emerg("gpio-%d (%s): " fmt, desc_to_gpio(desc), desc->label ? : "?",\ - ##__VA_ARGS__) -#define gpiod_crit(desc, fmt, ...) \ - pr_crit("gpio-%d (%s): " fmt, desc_to_gpio(desc), desc->label ? : "?", \ - ##__VA_ARGS__) -#define gpiod_err(desc, fmt, ...) \ - pr_err("gpio-%d (%s): " fmt, desc_to_gpio(desc), desc->label ? : "?", \ - ##__VA_ARGS__) -#define gpiod_warn(desc, fmt, ...) \ - pr_warn("gpio-%d (%s): " fmt, desc_to_gpio(desc), desc->label ? : "?", \ - ##__VA_ARGS__) -#define gpiod_info(desc, fmt, ...) \ - pr_info("gpio-%d (%s): " fmt, desc_to_gpio(desc), desc->label ? : "?", \ - ##__VA_ARGS__) -#define gpiod_dbg(desc, fmt, ...) \ - pr_debug("gpio-%d (%s): " fmt, desc_to_gpio(desc), desc->label ? : "?",\ - ##__VA_ARGS__) +#define gpiod_err(desc, fmt, ...) \ +do { \ + scoped_guard(srcu, &desc->srcu) { \ + pr_err("gpio-%d (%s): " fmt, desc_to_gpio(desc), \ + gpiod_get_label(desc) ? : "?", ##__VA_ARGS__); \ + } \ +} while (0) + +#define gpiod_warn(desc, fmt, ...) \ +do { \ + scoped_guard(srcu, &desc->srcu) { \ + pr_warn("gpio-%d (%s): " fmt, desc_to_gpio(desc), \ + gpiod_get_label(desc) ? : "?", ##__VA_ARGS__); \ + } \ +} while (0) + +#define gpiod_dbg(desc, fmt, ...) \ +do { \ + scoped_guard(srcu, &desc->srcu) { \ + pr_debug("gpio-%d (%s): " fmt, desc_to_gpio(desc), \ + gpiod_get_label(desc) ? : "?", ##__VA_ARGS__); \ + } \ +} while (0) /* With chip prefix */ -#define chip_emerg(gc, fmt, ...) \ - dev_emerg(&gc->gpiodev->dev, "(%s): " fmt, gc->label, ##__VA_ARGS__) -#define chip_crit(gc, fmt, ...) \ - dev_crit(&gc->gpiodev->dev, "(%s): " fmt, gc->label, ##__VA_ARGS__) #define chip_err(gc, fmt, ...) \ dev_err(&gc->gpiodev->dev, "(%s): " fmt, gc->label, ##__VA_ARGS__) #define chip_warn(gc, fmt, ...) \ diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi86.c b/drivers/gpu/drm/bridge/ti-sn65dsi86.c index 62cc3893dca5..7fbc307cc025 100644 --- a/drivers/gpu/drm/bridge/ti-sn65dsi86.c +++ b/drivers/gpu/drm/bridge/ti-sn65dsi86.c @@ -197,7 +197,7 @@ struct ti_sn65dsi86 { DECLARE_BITMAP(gchip_output, SN_NUM_GPIOS); #endif #if defined(CONFIG_PWM) - struct pwm_chip pchip; + struct pwm_chip *pchip; bool pwm_enabled; atomic_t pwm_pin_busy; #endif @@ -1374,7 +1374,7 @@ static void ti_sn_pwm_pin_release(struct ti_sn65dsi86 *pdata) static struct ti_sn65dsi86 *pwm_chip_to_ti_sn_bridge(struct pwm_chip *chip) { - return container_of(chip, struct ti_sn65dsi86, pchip); + return pwmchip_get_drvdata(chip); } static int ti_sn_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) @@ -1415,7 +1415,7 @@ static int ti_sn_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, int ret; if (!pdata->pwm_enabled) { - ret = pm_runtime_resume_and_get(chip->dev); + ret = pm_runtime_resume_and_get(pwmchip_parent(chip)); if (ret < 0) return ret; } @@ -1431,7 +1431,7 @@ static int ti_sn_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, SN_GPIO_MUX_MASK << (2 * SN_PWM_GPIO_IDX), SN_GPIO_MUX_SPECIAL << (2 * SN_PWM_GPIO_IDX)); if (ret) { - dev_err(chip->dev, "failed to mux in PWM function\n"); + dev_err(pwmchip_parent(chip), "failed to mux in PWM function\n"); goto out; } } @@ -1507,7 +1507,7 @@ static int ti_sn_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, ret = regmap_write(pdata->regmap, SN_PWM_PRE_DIV_REG, pre_div); if (ret) { - dev_err(chip->dev, "failed to update PWM_PRE_DIV\n"); + dev_err(pwmchip_parent(chip), "failed to update PWM_PRE_DIV\n"); goto out; } @@ -1519,7 +1519,7 @@ static int ti_sn_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, FIELD_PREP(SN_PWM_INV_MASK, state->polarity == PWM_POLARITY_INVERSED); ret = regmap_write(pdata->regmap, SN_PWM_EN_INV_REG, pwm_en_inv); if (ret) { - dev_err(chip->dev, "failed to update PWM_EN/PWM_INV\n"); + dev_err(pwmchip_parent(chip), "failed to update PWM_EN/PWM_INV\n"); goto out; } @@ -1527,7 +1527,7 @@ static int ti_sn_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, out: if (!pdata->pwm_enabled) - pm_runtime_put_sync(chip->dev); + pm_runtime_put_sync(pwmchip_parent(chip)); return ret; } @@ -1585,24 +1585,28 @@ static const struct pwm_ops ti_sn_pwm_ops = { static int ti_sn_pwm_probe(struct auxiliary_device *adev, const struct auxiliary_device_id *id) { + struct pwm_chip *chip; struct ti_sn65dsi86 *pdata = dev_get_drvdata(adev->dev.parent); - pdata->pchip.dev = &adev->dev; - pdata->pchip.ops = &ti_sn_pwm_ops; - pdata->pchip.npwm = 1; - pdata->pchip.of_xlate = of_pwm_single_xlate; - pdata->pchip.of_pwm_n_cells = 1; + pdata->pchip = chip = devm_pwmchip_alloc(&adev->dev, 1, 0); + if (IS_ERR(chip)) + return PTR_ERR(chip); + + pwmchip_set_drvdata(chip, pdata); + + chip->ops = &ti_sn_pwm_ops; + chip->of_xlate = of_pwm_single_xlate; devm_pm_runtime_enable(&adev->dev); - return pwmchip_add(&pdata->pchip); + return pwmchip_add(chip); } static void ti_sn_pwm_remove(struct auxiliary_device *adev) { struct ti_sn65dsi86 *pdata = dev_get_drvdata(adev->dev.parent); - pwmchip_remove(&pdata->pchip); + pwmchip_remove(pdata->pchip); if (pdata->pwm_enabled) pm_runtime_put_sync(&adev->dev); diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c index 860b51b56a92..d4e844128826 100644 --- a/drivers/gpu/drm/i915/intel_runtime_pm.c +++ b/drivers/gpu/drm/i915/intel_runtime_pm.c @@ -246,7 +246,10 @@ static intel_wakeref_t __intel_runtime_pm_get_if_active(struct intel_runtime_pm * function, since the power state is undefined. This applies * atm to the late/early system suspend/resume handlers. */ - if (pm_runtime_get_if_active(rpm->kdev, ignore_usecount) <= 0) + if ((ignore_usecount && + pm_runtime_get_if_active(rpm->kdev) <= 0) || + (!ignore_usecount && + pm_runtime_get_if_in_use(rpm->kdev) <= 0)) return 0; } diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c index b429c2876a76..dd110058bf74 100644 --- a/drivers/gpu/drm/xe/xe_pm.c +++ b/drivers/gpu/drm/xe/xe_pm.c @@ -330,7 +330,7 @@ int xe_pm_runtime_put(struct xe_device *xe) int xe_pm_runtime_get_if_active(struct xe_device *xe) { - return pm_runtime_get_if_active(xe->drm.dev, true); + return pm_runtime_get_if_active(xe->drm.dev); } void xe_pm_assert_unbounded_bridge(struct xe_device *xe) diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index a608264da87d..83945397b6eb 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -301,6 +301,16 @@ config SENSORS_ASC7621 This driver can also be built as a module. If so, the module will be called asc7621. +config SENSORS_ASUS_ROG_RYUJIN + tristate "ASUS ROG RYUJIN II 360 hardware monitoring driver" + depends on HID + help + If you say yes here you get support for the fans and sensors of + the ASUS ROG RYUJIN II 360 AIO CPU liquid cooler. + + This driver can also be built as a module. If so, the module + will be called asus_rog_ryujin. + config SENSORS_AXI_FAN_CONTROL tristate "Analog Devices FAN Control HDL Core driver" help @@ -412,6 +422,17 @@ config SENSORS_ASPEED This driver can also be built as a module. If so, the module will be called aspeed_pwm_tacho. +config SENSORS_ASPEED_G6 + tristate "ASPEED g6 PWM and Fan tach driver" + depends on ARCH_ASPEED || COMPILE_TEST + depends on PWM + help + This driver provides support for ASPEED G6 PWM and Fan Tach + controllers. + + This driver can also be built as a module. If so, the module + will be called aspeed_pwm_tacho. + config SENSORS_ATXP1 tristate "Attansic ATXP1 VID controller" depends on I2C @@ -452,6 +473,16 @@ config SENSORS_BT1_PVT_ALARMS the data conversion will be periodically performed and the data will be saved in the internal driver cache. +config SENSORS_CHIPCAP2 + tristate "Amphenol ChipCap 2 relative humidity and temperature sensor" + depends on I2C + help + Say yes here to build support for the Amphenol ChipCap 2 + relative humidity and temperature sensor. + + To compile this driver as a module, choose M here: the module + will be called chipcap2. + config SENSORS_CORSAIR_CPRO tristate "Corsair Commander Pro controller" depends on HID @@ -1038,6 +1069,17 @@ config SENSORS_LTC4261 This driver can also be built as a module. If so, the module will be called ltc4261. +config SENSORS_LTC4282 + tristate "Analog Devices LTC4282" + depends on I2C + select REGMAP_I2C + help + If you say yes here you get support for Analog Devices LTC4282 + High Current Hot Swap Controller I2C interface. + + This driver can also be built as a module. If so, the module will + be called ltc4282. + config SENSORS_LTQ_CPUTEMP bool "Lantiq cpu temperature sensor driver" depends on SOC_XWAY @@ -1674,6 +1716,16 @@ config SENSORS_NZXT_KRAKEN2 This driver can also be built as a module. If so, the module will be called nzxt-kraken2. +config SENSORS_NZXT_KRAKEN3 + tristate "NZXT Kraken X53/X63/X73, Z53/Z63/Z73 coolers" + depends on USB_HID + help + If you say yes here you get support for hardware monitoring for the + NZXT Kraken X53/X63/X73, Z53/Z63/Z73 all-in-one CPU liquid coolers. + + This driver can also be built as a module. If so, the module + will be called nzxt-kraken3. + config SENSORS_NZXT_SMART2 tristate "NZXT RGB & Fan Controller/Smart Device v2" depends on USB_HID @@ -1714,6 +1766,16 @@ source "drivers/hwmon/peci/Kconfig" source "drivers/hwmon/pmbus/Kconfig" +config SENSORS_PT5161L + tristate "Astera Labs PT5161L PCIe retimer hardware monitoring" + depends on I2C + help + If you say yes here you get support for temperature monitoring + on the Astera Labs PT5161L PCIe retimer. + + This driver can also be built as a module. If so, the module + will be called pt5161l. + config SENSORS_PWM_FAN tristate "PWM fan" depends on (PWM && OF) || COMPILE_TEST @@ -1994,6 +2056,20 @@ config SENSORS_SFCTEMP This driver can also be built as a module. If so, the module will be called sfctemp. +config SENSORS_SURFACE_FAN + tristate "Surface Fan Driver" + depends on SURFACE_AGGREGATOR + depends on SURFACE_AGGREGATOR_BUS + help + Driver that provides monitoring of the fan on Surface Pro devices that + have a fan, like the Surface Pro 9. + + This makes the fan's current speed accessible through the hwmon + system. It does not provide control over the fan, the firmware is + responsible for that, this driver merely provides monitoring. + + Select M or Y here, if you want to be able to read the fan's speed. + config SENSORS_ADC128D818 tristate "Texas Instruments ADC128D818" depends on I2C diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile index 47be39af5c03..5c31808f6378 100644 --- a/drivers/hwmon/Makefile +++ b/drivers/hwmon/Makefile @@ -55,9 +55,12 @@ obj-$(CONFIG_SENSORS_ARM_SCPI) += scpi-hwmon.o obj-$(CONFIG_SENSORS_AS370) += as370-hwmon.o obj-$(CONFIG_SENSORS_ASC7621) += asc7621.o obj-$(CONFIG_SENSORS_ASPEED) += aspeed-pwm-tacho.o +obj-$(CONFIG_SENSORS_ASPEED_G6) += aspeed-g6-pwm-tach.o +obj-$(CONFIG_SENSORS_ASUS_ROG_RYUJIN) += asus_rog_ryujin.o obj-$(CONFIG_SENSORS_ATXP1) += atxp1.o obj-$(CONFIG_SENSORS_AXI_FAN_CONTROL) += axi-fan-control.o obj-$(CONFIG_SENSORS_BT1_PVT) += bt1-pvt.o +obj-$(CONFIG_SENSORS_CHIPCAP2) += chipcap2.o obj-$(CONFIG_SENSORS_CORETEMP) += coretemp.o obj-$(CONFIG_SENSORS_CORSAIR_CPRO) += corsair-cpro.o obj-$(CONFIG_SENSORS_CORSAIR_PSU) += corsair-psu.o @@ -136,6 +139,7 @@ obj-$(CONFIG_SENSORS_LTC4222) += ltc4222.o obj-$(CONFIG_SENSORS_LTC4245) += ltc4245.o obj-$(CONFIG_SENSORS_LTC4260) += ltc4260.o obj-$(CONFIG_SENSORS_LTC4261) += ltc4261.o +obj-$(CONFIG_SENSORS_LTC4282) += ltc4282.o obj-$(CONFIG_SENSORS_LTQ_CPUTEMP) += ltq-cputemp.o obj-$(CONFIG_SENSORS_MAX1111) += max1111.o obj-$(CONFIG_SENSORS_MAX127) += max127.o @@ -173,6 +177,7 @@ obj-$(CONFIG_SENSORS_NPCM7XX) += npcm750-pwm-fan.o obj-$(CONFIG_SENSORS_NSA320) += nsa320-hwmon.o obj-$(CONFIG_SENSORS_NTC_THERMISTOR) += ntc_thermistor.o obj-$(CONFIG_SENSORS_NZXT_KRAKEN2) += nzxt-kraken2.o +obj-$(CONFIG_SENSORS_NZXT_KRAKEN3) += nzxt-kraken3.o obj-$(CONFIG_SENSORS_NZXT_SMART2) += nzxt-smart2.o obj-$(CONFIG_SENSORS_OXP) += oxp-sensors.o obj-$(CONFIG_SENSORS_PC87360) += pc87360.o @@ -180,6 +185,7 @@ obj-$(CONFIG_SENSORS_PC87427) += pc87427.o obj-$(CONFIG_SENSORS_PCF8591) += pcf8591.o obj-$(CONFIG_SENSORS_POWERZ) += powerz.o obj-$(CONFIG_SENSORS_POWR1220) += powr1220.o +obj-$(CONFIG_SENSORS_PT5161L) += pt5161l.o obj-$(CONFIG_SENSORS_PWM_FAN) += pwm-fan.o obj-$(CONFIG_SENSORS_RASPBERRYPI_HWMON) += raspberrypi-hwmon.o obj-$(CONFIG_SENSORS_SBTSI) += sbtsi_temp.o @@ -201,6 +207,7 @@ obj-$(CONFIG_SENSORS_SMSC47M1) += smsc47m1.o obj-$(CONFIG_SENSORS_SMSC47M192)+= smsc47m192.o obj-$(CONFIG_SENSORS_SPARX5) += sparx5-temp.o obj-$(CONFIG_SENSORS_STTS751) += stts751.o +obj-$(CONFIG_SENSORS_SURFACE_FAN)+= surface_fan.o obj-$(CONFIG_SENSORS_SY7636A) += sy7636a-hwmon.o obj-$(CONFIG_SENSORS_AMC6821) += amc6821.o obj-$(CONFIG_SENSORS_TC74) += tc74.o diff --git a/drivers/hwmon/adm1177.c b/drivers/hwmon/adm1177.c index 60a893f27159..3390102d2d4a 100644 --- a/drivers/hwmon/adm1177.c +++ b/drivers/hwmon/adm1177.c @@ -250,7 +250,6 @@ static const struct of_device_id adm1177_dt_ids[] = { MODULE_DEVICE_TABLE(of, adm1177_dt_ids); static struct i2c_driver adm1177_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "adm1177", .of_match_table = adm1177_dt_ids, diff --git a/drivers/hwmon/adt7310.c b/drivers/hwmon/adt7310.c index 067865f4887a..25281739aa3b 100644 --- a/drivers/hwmon/adt7310.c +++ b/drivers/hwmon/adt7310.c @@ -124,7 +124,7 @@ static int adt7310_reg_write(void *context, unsigned int reg, unsigned int val) static const struct regmap_config adt7310_regmap_config = { .reg_bits = 8, .val_bits = 16, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, .volatile_reg = adt7310_regmap_is_volatile, .reg_read = adt7310_reg_read, .reg_write = adt7310_reg_write, diff --git a/drivers/hwmon/adt7410.c b/drivers/hwmon/adt7410.c index 952506779336..d15f64d4b6e7 100644 --- a/drivers/hwmon/adt7410.c +++ b/drivers/hwmon/adt7410.c @@ -69,7 +69,7 @@ static const struct regmap_config adt7410_regmap_config = { .reg_bits = 8, .val_bits = 16, .max_register = ADT7X10_ID, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, .volatile_reg = adt7410_regmap_is_volatile, .reg_read = adt7410_reg_read, .reg_write = adt7410_reg_write, @@ -95,14 +95,12 @@ static const struct i2c_device_id adt7410_ids[] = { MODULE_DEVICE_TABLE(i2c, adt7410_ids); static struct i2c_driver adt7410_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "adt7410", .pm = pm_sleep_ptr(&adt7x10_dev_pm_ops), }, .probe = adt7410_i2c_probe, .id_table = adt7410_ids, - .address_list = I2C_ADDRS(0x48, 0x49, 0x4a, 0x4b), }; module_i2c_driver(adt7410_driver); diff --git a/drivers/hwmon/amc6821.c b/drivers/hwmon/amc6821.c index 2a7a4b6b0094..9b02b304c2f5 100644 --- a/drivers/hwmon/amc6821.c +++ b/drivers/hwmon/amc6821.c @@ -934,10 +934,21 @@ static const struct i2c_device_id amc6821_id[] = { MODULE_DEVICE_TABLE(i2c, amc6821_id); +static const struct of_device_id __maybe_unused amc6821_of_match[] = { + { + .compatible = "ti,amc6821", + .data = (void *)amc6821, + }, + { } +}; + +MODULE_DEVICE_TABLE(of, amc6821_of_match); + static struct i2c_driver amc6821_driver = { .class = I2C_CLASS_HWMON, .driver = { .name = "amc6821", + .of_match_table = of_match_ptr(amc6821_of_match), }, .probe = amc6821_probe, .id_table = amc6821_id, diff --git a/drivers/hwmon/aspeed-g6-pwm-tach.c b/drivers/hwmon/aspeed-g6-pwm-tach.c new file mode 100644 index 000000000000..597b3b019d49 --- /dev/null +++ b/drivers/hwmon/aspeed-g6-pwm-tach.c @@ -0,0 +1,549 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2021 Aspeed Technology Inc. + * + * PWM/TACH controller driver for Aspeed ast2600 SoCs. + * This drivers doesn't support earlier version of the IP. + * + * The hardware operates in time quantities of length + * Q := (DIV_L + 1) << DIV_H / input-clk + * The length of a PWM period is (DUTY_CYCLE_PERIOD + 1) * Q. + * The maximal value for DUTY_CYCLE_PERIOD is used here to provide + * a fine grained selection for the duty cycle. + * + * This driver uses DUTY_CYCLE_RISING_POINT = 0, so from the start of a + * period the output is active until DUTY_CYCLE_FALLING_POINT * Q. Note + * that if DUTY_CYCLE_RISING_POINT = DUTY_CYCLE_FALLING_POINT the output is + * always active. + * + * Register usage: + * PIN_ENABLE: When it is unset the pwm controller will emit inactive level to the external. + * Use to determine whether the PWM channel is enabled or disabled + * CLK_ENABLE: When it is unset the pwm controller will assert the duty counter reset and + * emit inactive level to the PIN_ENABLE mux after that the driver can still change the pwm period + * and duty and the value will apply when CLK_ENABLE be set again. + * Use to determine whether duty_cycle bigger than 0. + * PWM_ASPEED_CTRL_INVERSE: When it is toggled the output value will inverse immediately. + * PWM_ASPEED_DUTY_CYCLE_FALLING_POINT/PWM_ASPEED_DUTY_CYCLE_RISING_POINT: When these two + * values are equal it means the duty cycle = 100%. + * + * The glitch may generate at: + * - Enabled changing when the duty_cycle bigger than 0% and less than 100%. + * - Polarity changing when the duty_cycle bigger than 0% and less than 100%. + * + * Limitations: + * - When changing both duty cycle and period, we cannot prevent in + * software that the output might produce a period with mixed + * settings. + * - Disabling the PWM doesn't complete the current period. + * + * Improvements: + * - When only changing one of duty cycle or period, our pwm controller will not + * generate the glitch, the configure will change at next cycle of pwm. + * This improvement can disable/enable through PWM_ASPEED_CTRL_DUTY_SYNC_DISABLE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* The channel number of Aspeed pwm controller */ +#define PWM_ASPEED_NR_PWMS 16 +/* PWM Control Register */ +#define PWM_ASPEED_CTRL(ch) ((ch) * 0x10 + 0x00) +#define PWM_ASPEED_CTRL_LOAD_SEL_RISING_AS_WDT BIT(19) +#define PWM_ASPEED_CTRL_DUTY_LOAD_AS_WDT_ENABLE BIT(18) +#define PWM_ASPEED_CTRL_DUTY_SYNC_DISABLE BIT(17) +#define PWM_ASPEED_CTRL_CLK_ENABLE BIT(16) +#define PWM_ASPEED_CTRL_LEVEL_OUTPUT BIT(15) +#define PWM_ASPEED_CTRL_INVERSE BIT(14) +#define PWM_ASPEED_CTRL_OPEN_DRAIN_ENABLE BIT(13) +#define PWM_ASPEED_CTRL_PIN_ENABLE BIT(12) +#define PWM_ASPEED_CTRL_CLK_DIV_H GENMASK(11, 8) +#define PWM_ASPEED_CTRL_CLK_DIV_L GENMASK(7, 0) + +/* PWM Duty Cycle Register */ +#define PWM_ASPEED_DUTY_CYCLE(ch) ((ch) * 0x10 + 0x04) +#define PWM_ASPEED_DUTY_CYCLE_PERIOD GENMASK(31, 24) +#define PWM_ASPEED_DUTY_CYCLE_POINT_AS_WDT GENMASK(23, 16) +#define PWM_ASPEED_DUTY_CYCLE_FALLING_POINT GENMASK(15, 8) +#define PWM_ASPEED_DUTY_CYCLE_RISING_POINT GENMASK(7, 0) + +/* PWM fixed value */ +#define PWM_ASPEED_FIXED_PERIOD FIELD_MAX(PWM_ASPEED_DUTY_CYCLE_PERIOD) + +/* The channel number of Aspeed tach controller */ +#define TACH_ASPEED_NR_TACHS 16 +/* TACH Control Register */ +#define TACH_ASPEED_CTRL(ch) (((ch) * 0x10) + 0x08) +#define TACH_ASPEED_IER BIT(31) +#define TACH_ASPEED_INVERS_LIMIT BIT(30) +#define TACH_ASPEED_LOOPBACK BIT(29) +#define TACH_ASPEED_ENABLE BIT(28) +#define TACH_ASPEED_DEBOUNCE_MASK GENMASK(27, 26) +#define TACH_ASPEED_DEBOUNCE_BIT 26 +#define TACH_ASPEED_IO_EDGE_MASK GENMASK(25, 24) +#define TACH_ASPEED_IO_EDGE_BIT 24 +#define TACH_ASPEED_CLK_DIV_T_MASK GENMASK(23, 20) +#define TACH_ASPEED_CLK_DIV_BIT 20 +#define TACH_ASPEED_THRESHOLD_MASK GENMASK(19, 0) +/* [27:26] */ +#define DEBOUNCE_3_CLK 0x00 +#define DEBOUNCE_2_CLK 0x01 +#define DEBOUNCE_1_CLK 0x02 +#define DEBOUNCE_0_CLK 0x03 +/* [25:24] */ +#define F2F_EDGES 0x00 +#define R2R_EDGES 0x01 +#define BOTH_EDGES 0x02 +/* [23:20] */ +/* divisor = 4 to the nth power, n = register value */ +#define DEFAULT_TACH_DIV 1024 +#define DIV_TO_REG(divisor) (ilog2(divisor) >> 1) + +/* TACH Status Register */ +#define TACH_ASPEED_STS(ch) (((ch) * 0x10) + 0x0C) + +/*PWM_TACH_STS */ +#define TACH_ASPEED_ISR BIT(31) +#define TACH_ASPEED_PWM_OUT BIT(25) +#define TACH_ASPEED_PWM_OEN BIT(24) +#define TACH_ASPEED_DEB_INPUT BIT(23) +#define TACH_ASPEED_RAW_INPUT BIT(22) +#define TACH_ASPEED_VALUE_UPDATE BIT(21) +#define TACH_ASPEED_FULL_MEASUREMENT BIT(20) +#define TACH_ASPEED_VALUE_MASK GENMASK(19, 0) +/********************************************************** + * Software setting + *********************************************************/ +#define DEFAULT_FAN_PULSE_PR 2 + +struct aspeed_pwm_tach_data { + struct device *dev; + void __iomem *base; + struct clk *clk; + struct reset_control *reset; + unsigned long clk_rate; + struct pwm_chip chip; + bool tach_present[TACH_ASPEED_NR_TACHS]; + u32 tach_divisor; +}; + +static inline struct aspeed_pwm_tach_data * +aspeed_pwm_chip_to_data(struct pwm_chip *chip) +{ + return container_of(chip, struct aspeed_pwm_tach_data, chip); +} + +static int aspeed_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, + struct pwm_state *state) +{ + struct aspeed_pwm_tach_data *priv = aspeed_pwm_chip_to_data(chip); + u32 hwpwm = pwm->hwpwm; + bool polarity, pin_en, clk_en; + u32 duty_pt, val; + u64 div_h, div_l, duty_cycle_period, dividend; + + val = readl(priv->base + PWM_ASPEED_CTRL(hwpwm)); + polarity = FIELD_GET(PWM_ASPEED_CTRL_INVERSE, val); + pin_en = FIELD_GET(PWM_ASPEED_CTRL_PIN_ENABLE, val); + clk_en = FIELD_GET(PWM_ASPEED_CTRL_CLK_ENABLE, val); + div_h = FIELD_GET(PWM_ASPEED_CTRL_CLK_DIV_H, val); + div_l = FIELD_GET(PWM_ASPEED_CTRL_CLK_DIV_L, val); + val = readl(priv->base + PWM_ASPEED_DUTY_CYCLE(hwpwm)); + duty_pt = FIELD_GET(PWM_ASPEED_DUTY_CYCLE_FALLING_POINT, val); + duty_cycle_period = FIELD_GET(PWM_ASPEED_DUTY_CYCLE_PERIOD, val); + /* + * This multiplication doesn't overflow, the upper bound is + * 1000000000 * 256 * 256 << 15 = 0x1dcd650000000000 + */ + dividend = (u64)NSEC_PER_SEC * (div_l + 1) * (duty_cycle_period + 1) + << div_h; + state->period = DIV_ROUND_UP_ULL(dividend, priv->clk_rate); + + if (clk_en && duty_pt) { + dividend = (u64)NSEC_PER_SEC * (div_l + 1) * duty_pt + << div_h; + state->duty_cycle = DIV_ROUND_UP_ULL(dividend, priv->clk_rate); + } else { + state->duty_cycle = clk_en ? state->period : 0; + } + state->polarity = polarity ? PWM_POLARITY_INVERSED : PWM_POLARITY_NORMAL; + state->enabled = pin_en; + return 0; +} + +static int aspeed_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, + const struct pwm_state *state) +{ + struct aspeed_pwm_tach_data *priv = aspeed_pwm_chip_to_data(chip); + u32 hwpwm = pwm->hwpwm, duty_pt, val; + u64 div_h, div_l, divisor, expect_period; + bool clk_en; + + expect_period = div64_u64(ULLONG_MAX, (u64)priv->clk_rate); + expect_period = min(expect_period, state->period); + dev_dbg(chip->dev, "expect period: %lldns, duty_cycle: %lldns", + expect_period, state->duty_cycle); + /* + * Pick the smallest value for div_h so that div_l can be the biggest + * which results in a finer resolution near the target period value. + */ + divisor = (u64)NSEC_PER_SEC * (PWM_ASPEED_FIXED_PERIOD + 1) * + (FIELD_MAX(PWM_ASPEED_CTRL_CLK_DIV_L) + 1); + div_h = order_base_2(DIV64_U64_ROUND_UP(priv->clk_rate * expect_period, divisor)); + if (div_h > 0xf) + div_h = 0xf; + + divisor = ((u64)NSEC_PER_SEC * (PWM_ASPEED_FIXED_PERIOD + 1)) << div_h; + div_l = div64_u64(priv->clk_rate * expect_period, divisor); + + if (div_l == 0) + return -ERANGE; + + div_l -= 1; + + if (div_l > 255) + div_l = 255; + + dev_dbg(chip->dev, "clk source: %ld div_h %lld, div_l : %lld\n", + priv->clk_rate, div_h, div_l); + /* duty_pt = duty_cycle * (PERIOD + 1) / period */ + duty_pt = div64_u64(state->duty_cycle * priv->clk_rate, + (u64)NSEC_PER_SEC * (div_l + 1) << div_h); + dev_dbg(chip->dev, "duty_cycle = %lld, duty_pt = %d\n", + state->duty_cycle, duty_pt); + + /* + * Fixed DUTY_CYCLE_PERIOD to its max value to get a + * fine-grained resolution for duty_cycle at the expense of a + * coarser period resolution. + */ + val = readl(priv->base + PWM_ASPEED_DUTY_CYCLE(hwpwm)); + val &= ~PWM_ASPEED_DUTY_CYCLE_PERIOD; + val |= FIELD_PREP(PWM_ASPEED_DUTY_CYCLE_PERIOD, + PWM_ASPEED_FIXED_PERIOD); + writel(val, priv->base + PWM_ASPEED_DUTY_CYCLE(hwpwm)); + + if (duty_pt == 0) { + /* emit inactive level and assert the duty counter reset */ + clk_en = 0; + } else { + clk_en = 1; + if (duty_pt >= (PWM_ASPEED_FIXED_PERIOD + 1)) + duty_pt = 0; + val = readl(priv->base + PWM_ASPEED_DUTY_CYCLE(hwpwm)); + val &= ~(PWM_ASPEED_DUTY_CYCLE_RISING_POINT | + PWM_ASPEED_DUTY_CYCLE_FALLING_POINT); + val |= FIELD_PREP(PWM_ASPEED_DUTY_CYCLE_FALLING_POINT, duty_pt); + writel(val, priv->base + PWM_ASPEED_DUTY_CYCLE(hwpwm)); + } + + val = readl(priv->base + PWM_ASPEED_CTRL(hwpwm)); + val &= ~(PWM_ASPEED_CTRL_CLK_DIV_H | PWM_ASPEED_CTRL_CLK_DIV_L | + PWM_ASPEED_CTRL_PIN_ENABLE | PWM_ASPEED_CTRL_CLK_ENABLE | + PWM_ASPEED_CTRL_INVERSE); + val |= FIELD_PREP(PWM_ASPEED_CTRL_CLK_DIV_H, div_h) | + FIELD_PREP(PWM_ASPEED_CTRL_CLK_DIV_L, div_l) | + FIELD_PREP(PWM_ASPEED_CTRL_PIN_ENABLE, state->enabled) | + FIELD_PREP(PWM_ASPEED_CTRL_CLK_ENABLE, clk_en) | + FIELD_PREP(PWM_ASPEED_CTRL_INVERSE, state->polarity); + writel(val, priv->base + PWM_ASPEED_CTRL(hwpwm)); + + return 0; +} + +static const struct pwm_ops aspeed_pwm_ops = { + .apply = aspeed_pwm_apply, + .get_state = aspeed_pwm_get_state, +}; + +static void aspeed_tach_ch_enable(struct aspeed_pwm_tach_data *priv, u8 tach_ch, + bool enable) +{ + if (enable) + writel(readl(priv->base + TACH_ASPEED_CTRL(tach_ch)) | + TACH_ASPEED_ENABLE, + priv->base + TACH_ASPEED_CTRL(tach_ch)); + else + writel(readl(priv->base + TACH_ASPEED_CTRL(tach_ch)) & + ~TACH_ASPEED_ENABLE, + priv->base + TACH_ASPEED_CTRL(tach_ch)); +} + +static int aspeed_tach_val_to_rpm(struct aspeed_pwm_tach_data *priv, u32 tach_val) +{ + u64 rpm; + u32 tach_div; + + tach_div = tach_val * priv->tach_divisor * DEFAULT_FAN_PULSE_PR; + + dev_dbg(priv->dev, "clk %ld, tach_val %d , tach_div %d\n", + priv->clk_rate, tach_val, tach_div); + + rpm = (u64)priv->clk_rate * 60; + do_div(rpm, tach_div); + + return (int)rpm; +} + +static int aspeed_get_fan_tach_ch_rpm(struct aspeed_pwm_tach_data *priv, + u8 fan_tach_ch) +{ + u32 val; + + val = readl(priv->base + TACH_ASPEED_STS(fan_tach_ch)); + + if (!(val & TACH_ASPEED_FULL_MEASUREMENT)) + return 0; + val = FIELD_GET(TACH_ASPEED_VALUE_MASK, val); + return aspeed_tach_val_to_rpm(priv, val); +} + +static int aspeed_tach_hwmon_read(struct device *dev, + enum hwmon_sensor_types type, u32 attr, + int channel, long *val) +{ + struct aspeed_pwm_tach_data *priv = dev_get_drvdata(dev); + u32 reg_val; + + switch (attr) { + case hwmon_fan_input: + *val = aspeed_get_fan_tach_ch_rpm(priv, channel); + break; + case hwmon_fan_div: + reg_val = readl(priv->base + TACH_ASPEED_CTRL(channel)); + reg_val = FIELD_GET(TACH_ASPEED_CLK_DIV_T_MASK, reg_val); + *val = BIT(reg_val << 1); + break; + default: + return -EOPNOTSUPP; + } + return 0; +} + +static int aspeed_tach_hwmon_write(struct device *dev, + enum hwmon_sensor_types type, u32 attr, + int channel, long val) +{ + struct aspeed_pwm_tach_data *priv = dev_get_drvdata(dev); + u32 reg_val; + + switch (attr) { + case hwmon_fan_div: + if (!is_power_of_2(val) || (ilog2(val) % 2) || + DIV_TO_REG(val) > 0xb) + return -EINVAL; + priv->tach_divisor = val; + reg_val = readl(priv->base + TACH_ASPEED_CTRL(channel)); + reg_val &= ~TACH_ASPEED_CLK_DIV_T_MASK; + reg_val |= FIELD_PREP(TACH_ASPEED_CLK_DIV_T_MASK, + DIV_TO_REG(priv->tach_divisor)); + writel(reg_val, priv->base + TACH_ASPEED_CTRL(channel)); + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + +static umode_t aspeed_tach_dev_is_visible(const void *drvdata, + enum hwmon_sensor_types type, + u32 attr, int channel) +{ + const struct aspeed_pwm_tach_data *priv = drvdata; + + if (!priv->tach_present[channel]) + return 0; + switch (attr) { + case hwmon_fan_input: + return 0444; + case hwmon_fan_div: + return 0644; + } + return 0; +} + +static const struct hwmon_ops aspeed_tach_ops = { + .is_visible = aspeed_tach_dev_is_visible, + .read = aspeed_tach_hwmon_read, + .write = aspeed_tach_hwmon_write, +}; + +static const struct hwmon_channel_info *aspeed_tach_info[] = { + HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT | HWMON_F_DIV, HWMON_F_INPUT | HWMON_F_DIV, + HWMON_F_INPUT | HWMON_F_DIV, HWMON_F_INPUT | HWMON_F_DIV, + HWMON_F_INPUT | HWMON_F_DIV, HWMON_F_INPUT | HWMON_F_DIV, + HWMON_F_INPUT | HWMON_F_DIV, HWMON_F_INPUT | HWMON_F_DIV, + HWMON_F_INPUT | HWMON_F_DIV, HWMON_F_INPUT | HWMON_F_DIV, + HWMON_F_INPUT | HWMON_F_DIV, HWMON_F_INPUT | HWMON_F_DIV, + HWMON_F_INPUT | HWMON_F_DIV, HWMON_F_INPUT | HWMON_F_DIV, + HWMON_F_INPUT | HWMON_F_DIV, HWMON_F_INPUT | HWMON_F_DIV), + NULL +}; + +static const struct hwmon_chip_info aspeed_tach_chip_info = { + .ops = &aspeed_tach_ops, + .info = aspeed_tach_info, +}; + +static void aspeed_present_fan_tach(struct aspeed_pwm_tach_data *priv, u8 *tach_ch, int count) +{ + u8 ch, index; + u32 val; + + for (index = 0; index < count; index++) { + ch = tach_ch[index]; + priv->tach_present[ch] = true; + priv->tach_divisor = DEFAULT_TACH_DIV; + + val = readl(priv->base + TACH_ASPEED_CTRL(ch)); + val &= ~(TACH_ASPEED_INVERS_LIMIT | TACH_ASPEED_DEBOUNCE_MASK | + TACH_ASPEED_IO_EDGE_MASK | TACH_ASPEED_CLK_DIV_T_MASK | + TACH_ASPEED_THRESHOLD_MASK); + val |= (DEBOUNCE_3_CLK << TACH_ASPEED_DEBOUNCE_BIT) | + F2F_EDGES | + FIELD_PREP(TACH_ASPEED_CLK_DIV_T_MASK, + DIV_TO_REG(priv->tach_divisor)); + writel(val, priv->base + TACH_ASPEED_CTRL(ch)); + + aspeed_tach_ch_enable(priv, ch, true); + } +} + +static int aspeed_create_fan_monitor(struct device *dev, + struct device_node *child, + struct aspeed_pwm_tach_data *priv) +{ + int ret, count; + u8 *tach_ch; + + count = of_property_count_u8_elems(child, "tach-ch"); + if (count < 1) + return -EINVAL; + tach_ch = devm_kcalloc(dev, count, sizeof(*tach_ch), GFP_KERNEL); + if (!tach_ch) + return -ENOMEM; + ret = of_property_read_u8_array(child, "tach-ch", tach_ch, count); + if (ret) + return ret; + + aspeed_present_fan_tach(priv, tach_ch, count); + + return 0; +} + +static void aspeed_pwm_tach_reset_assert(void *data) +{ + struct reset_control *rst = data; + + reset_control_assert(rst); +} + +static int aspeed_pwm_tach_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev, *hwmon; + int ret; + struct device_node *child; + struct aspeed_pwm_tach_data *priv; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + priv->dev = dev; + priv->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(priv->base)) + return PTR_ERR(priv->base); + + priv->clk = devm_clk_get_enabled(dev, NULL); + if (IS_ERR(priv->clk)) + return dev_err_probe(dev, PTR_ERR(priv->clk), + "Couldn't get clock\n"); + priv->clk_rate = clk_get_rate(priv->clk); + priv->reset = devm_reset_control_get_exclusive(dev, NULL); + if (IS_ERR(priv->reset)) + return dev_err_probe(dev, PTR_ERR(priv->reset), + "Couldn't get reset control\n"); + + ret = reset_control_deassert(priv->reset); + if (ret) + return dev_err_probe(dev, ret, + "Couldn't deassert reset control\n"); + ret = devm_add_action_or_reset(dev, aspeed_pwm_tach_reset_assert, + priv->reset); + if (ret) + return ret; + + priv->chip.dev = dev; + priv->chip.ops = &aspeed_pwm_ops; + priv->chip.npwm = PWM_ASPEED_NR_PWMS; + + ret = devm_pwmchip_add(dev, &priv->chip); + if (ret) + return dev_err_probe(dev, ret, "Failed to add PWM chip\n"); + + for_each_child_of_node(dev->of_node, child) { + ret = aspeed_create_fan_monitor(dev, child, priv); + if (ret) { + of_node_put(child); + dev_warn(dev, "Failed to create fan %d", ret); + return 0; + } + } + + hwmon = devm_hwmon_device_register_with_info(dev, "aspeed_tach", priv, + &aspeed_tach_chip_info, NULL); + ret = PTR_ERR_OR_ZERO(hwmon); + if (ret) + return dev_err_probe(dev, ret, + "Failed to register hwmon device\n"); + + of_platform_populate(dev->of_node, NULL, NULL, dev); + + return 0; +} + +static int aspeed_pwm_tach_remove(struct platform_device *pdev) +{ + struct aspeed_pwm_tach_data *priv = platform_get_drvdata(pdev); + + reset_control_assert(priv->reset); + + return 0; +} + +static const struct of_device_id aspeed_pwm_tach_match[] = { + { + .compatible = "aspeed,ast2600-pwm-tach", + }, + {}, +}; +MODULE_DEVICE_TABLE(of, aspeed_pwm_tach_match); + +static struct platform_driver aspeed_pwm_tach_driver = { + .probe = aspeed_pwm_tach_probe, + .remove = aspeed_pwm_tach_remove, + .driver = { + .name = "aspeed-g6-pwm-tach", + .of_match_table = aspeed_pwm_tach_match, + }, +}; + +module_platform_driver(aspeed_pwm_tach_driver); + +MODULE_AUTHOR("Billy Tsai "); +MODULE_DESCRIPTION("Aspeed ast2600 PWM and Fan Tach device driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hwmon/asus_rog_ryujin.c b/drivers/hwmon/asus_rog_ryujin.c new file mode 100644 index 000000000000..f8b20346a995 --- /dev/null +++ b/drivers/hwmon/asus_rog_ryujin.c @@ -0,0 +1,609 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * hwmon driver for Asus ROG Ryujin II 360 AIO cooler. + * + * Copyright 2024 Aleksa Savic + */ + +#include +#include +#include +#include +#include +#include +#include + +#define DRIVER_NAME "asus_rog_ryujin" + +#define USB_VENDOR_ID_ASUS_ROG 0x0b05 +#define USB_PRODUCT_ID_RYUJIN_AIO 0x1988 /* ASUS ROG RYUJIN II 360 */ + +#define STATUS_VALIDITY 1500 /* ms */ +#define MAX_REPORT_LENGTH 65 + +/* Cooler status report offsets */ +#define RYUJIN_TEMP_SENSOR_1 3 +#define RYUJIN_TEMP_SENSOR_2 4 +#define RYUJIN_PUMP_SPEED 5 +#define RYUJIN_INTERNAL_FAN_SPEED 7 + +/* Cooler duty report offsets */ +#define RYUJIN_PUMP_DUTY 4 +#define RYUJIN_INTERNAL_FAN_DUTY 5 + +/* Controller status (speeds) report offsets */ +#define RYUJIN_CONTROLLER_SPEED_1 5 +#define RYUJIN_CONTROLLER_SPEED_2 7 +#define RYUJIN_CONTROLLER_SPEED_3 9 +#define RYUJIN_CONTROLLER_SPEED_4 3 + +/* Controller duty report offsets */ +#define RYUJIN_CONTROLLER_DUTY 4 + +/* Control commands and their inner offsets */ +#define RYUJIN_CMD_PREFIX 0xEC + +static const u8 get_cooler_status_cmd[] = { RYUJIN_CMD_PREFIX, 0x99 }; +static const u8 get_cooler_duty_cmd[] = { RYUJIN_CMD_PREFIX, 0x9A }; +static const u8 get_controller_speed_cmd[] = { RYUJIN_CMD_PREFIX, 0xA0 }; +static const u8 get_controller_duty_cmd[] = { RYUJIN_CMD_PREFIX, 0xA1 }; + +#define RYUJIN_SET_COOLER_PUMP_DUTY_OFFSET 3 +#define RYUJIN_SET_COOLER_FAN_DUTY_OFFSET 4 +static const u8 set_cooler_duty_cmd[] = { RYUJIN_CMD_PREFIX, 0x1A, 0x00, 0x00, 0x00 }; + +#define RYUJIN_SET_CONTROLLER_FAN_DUTY_OFFSET 4 +static const u8 set_controller_duty_cmd[] = { RYUJIN_CMD_PREFIX, 0x21, 0x00, 0x00, 0x00 }; + +/* Command lengths */ +#define GET_CMD_LENGTH 2 /* Same length for all get commands */ +#define SET_CMD_LENGTH 5 /* Same length for all set commands */ + +/* Command response headers */ +#define RYUJIN_GET_COOLER_STATUS_CMD_RESPONSE 0x19 +#define RYUJIN_GET_COOLER_DUTY_CMD_RESPONSE 0x1A +#define RYUJIN_GET_CONTROLLER_SPEED_CMD_RESPONSE 0x20 +#define RYUJIN_GET_CONTROLLER_DUTY_CMD_RESPONSE 0x21 + +static const char *const rog_ryujin_temp_label[] = { + "Coolant temp" +}; + +static const char *const rog_ryujin_speed_label[] = { + "Pump speed", + "Internal fan speed", + "Controller fan 1 speed", + "Controller fan 2 speed", + "Controller fan 3 speed", + "Controller fan 4 speed", +}; + +struct rog_ryujin_data { + struct hid_device *hdev; + struct device *hwmon_dev; + /* For locking access to buffer */ + struct mutex buffer_lock; + /* For queueing multiple readers */ + struct mutex status_report_request_mutex; + /* For reinitializing the completions below */ + spinlock_t status_report_request_lock; + struct completion cooler_status_received; + struct completion controller_status_received; + struct completion cooler_duty_received; + struct completion controller_duty_received; + struct completion cooler_duty_set; + struct completion controller_duty_set; + + /* Sensor data */ + s32 temp_input[1]; + u16 speed_input[6]; /* Pump, internal fan and four controller fan speeds in RPM */ + u8 duty_input[3]; /* Pump, internal fan and controller fan duty in PWM */ + + u8 *buffer; + unsigned long updated; /* jiffies */ +}; + +static int rog_ryujin_percent_to_pwm(u16 val) +{ + return DIV_ROUND_CLOSEST(val * 255, 100); +} + +static int rog_ryujin_pwm_to_percent(long val) +{ + return DIV_ROUND_CLOSEST(val * 100, 255); +} + +static umode_t rog_ryujin_is_visible(const void *data, + enum hwmon_sensor_types type, u32 attr, int channel) +{ + switch (type) { + case hwmon_temp: + switch (attr) { + case hwmon_temp_label: + case hwmon_temp_input: + return 0444; + default: + break; + } + break; + case hwmon_fan: + switch (attr) { + case hwmon_fan_label: + case hwmon_fan_input: + return 0444; + default: + break; + } + break; + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_input: + return 0644; + default: + break; + } + break; + default: + break; + } + + return 0; +} + +/* Writes the command to the device with the rest of the report filled with zeroes */ +static int rog_ryujin_write_expanded(struct rog_ryujin_data *priv, const u8 *cmd, int cmd_length) +{ + int ret; + + mutex_lock(&priv->buffer_lock); + + memcpy_and_pad(priv->buffer, MAX_REPORT_LENGTH, cmd, cmd_length, 0x00); + ret = hid_hw_output_report(priv->hdev, priv->buffer, MAX_REPORT_LENGTH); + + mutex_unlock(&priv->buffer_lock); + return ret; +} + +/* Assumes priv->status_report_request_mutex is locked */ +static int rog_ryujin_execute_cmd(struct rog_ryujin_data *priv, const u8 *cmd, int cmd_length, + struct completion *status_completion) +{ + int ret; + + /* + * Disable raw event parsing for a moment to safely reinitialize the + * completion. Reinit is done because hidraw could have triggered + * the raw event parsing and marked the passed in completion as done. + */ + spin_lock_bh(&priv->status_report_request_lock); + reinit_completion(status_completion); + spin_unlock_bh(&priv->status_report_request_lock); + + /* Send command for getting data */ + ret = rog_ryujin_write_expanded(priv, cmd, cmd_length); + if (ret < 0) + return ret; + + ret = wait_for_completion_interruptible_timeout(status_completion, + msecs_to_jiffies(STATUS_VALIDITY)); + if (ret == 0) + return -ETIMEDOUT; + else if (ret < 0) + return ret; + + return 0; +} + +static int rog_ryujin_get_status(struct rog_ryujin_data *priv) +{ + int ret = mutex_lock_interruptible(&priv->status_report_request_mutex); + + if (ret < 0) + return ret; + + if (!time_after(jiffies, priv->updated + msecs_to_jiffies(STATUS_VALIDITY))) { + /* Data is up to date */ + goto unlock_and_return; + } + + /* Retrieve cooler status */ + ret = + rog_ryujin_execute_cmd(priv, get_cooler_status_cmd, GET_CMD_LENGTH, + &priv->cooler_status_received); + if (ret < 0) + goto unlock_and_return; + + /* Retrieve controller status (speeds) */ + ret = + rog_ryujin_execute_cmd(priv, get_controller_speed_cmd, GET_CMD_LENGTH, + &priv->controller_status_received); + if (ret < 0) + goto unlock_and_return; + + /* Retrieve cooler duty */ + ret = + rog_ryujin_execute_cmd(priv, get_cooler_duty_cmd, GET_CMD_LENGTH, + &priv->cooler_duty_received); + if (ret < 0) + goto unlock_and_return; + + /* Retrieve controller duty */ + ret = + rog_ryujin_execute_cmd(priv, get_controller_duty_cmd, GET_CMD_LENGTH, + &priv->controller_duty_received); + if (ret < 0) + goto unlock_and_return; + + priv->updated = jiffies; + +unlock_and_return: + mutex_unlock(&priv->status_report_request_mutex); + if (ret < 0) + return ret; + + return 0; +} + +static int rog_ryujin_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *val) +{ + struct rog_ryujin_data *priv = dev_get_drvdata(dev); + int ret = rog_ryujin_get_status(priv); + + if (ret < 0) + return ret; + + switch (type) { + case hwmon_temp: + *val = priv->temp_input[channel]; + break; + case hwmon_fan: + *val = priv->speed_input[channel]; + break; + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_input: + *val = priv->duty_input[channel]; + break; + default: + return -EOPNOTSUPP; + } + break; + default: + return -EOPNOTSUPP; /* unreachable */ + } + + return 0; +} + +static int rog_ryujin_read_string(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, const char **str) +{ + switch (type) { + case hwmon_temp: + *str = rog_ryujin_temp_label[channel]; + break; + case hwmon_fan: + *str = rog_ryujin_speed_label[channel]; + break; + default: + return -EOPNOTSUPP; /* unreachable */ + } + + return 0; +} + +static int rog_ryujin_write_fixed_duty(struct rog_ryujin_data *priv, int channel, int val) +{ + u8 set_cmd[SET_CMD_LENGTH]; + int ret; + + if (channel < 2) { + /* + * Retrieve cooler duty since both pump and internal fan are set + * together, then write back with one of them modified. + */ + ret = mutex_lock_interruptible(&priv->status_report_request_mutex); + if (ret < 0) + return ret; + ret = + rog_ryujin_execute_cmd(priv, get_cooler_duty_cmd, GET_CMD_LENGTH, + &priv->cooler_duty_received); + if (ret < 0) + goto unlock_and_return; + + memcpy(set_cmd, set_cooler_duty_cmd, SET_CMD_LENGTH); + + /* Cooler duties are set as 0-100% */ + val = rog_ryujin_pwm_to_percent(val); + + if (channel == 0) { + /* Cooler pump duty */ + set_cmd[RYUJIN_SET_COOLER_PUMP_DUTY_OFFSET] = val; + set_cmd[RYUJIN_SET_COOLER_FAN_DUTY_OFFSET] = + rog_ryujin_pwm_to_percent(priv->duty_input[1]); + } else if (channel == 1) { + /* Cooler internal fan duty */ + set_cmd[RYUJIN_SET_COOLER_PUMP_DUTY_OFFSET] = + rog_ryujin_pwm_to_percent(priv->duty_input[0]); + set_cmd[RYUJIN_SET_COOLER_FAN_DUTY_OFFSET] = val; + } + + ret = rog_ryujin_execute_cmd(priv, set_cmd, SET_CMD_LENGTH, &priv->cooler_duty_set); +unlock_and_return: + mutex_unlock(&priv->status_report_request_mutex); + if (ret < 0) + return ret; + } else { + /* + * Controller fan duty (channel == 2). No need to retrieve current + * duty, so just send the command. + */ + memcpy(set_cmd, set_controller_duty_cmd, SET_CMD_LENGTH); + set_cmd[RYUJIN_SET_CONTROLLER_FAN_DUTY_OFFSET] = val; + + ret = + rog_ryujin_execute_cmd(priv, set_cmd, SET_CMD_LENGTH, + &priv->controller_duty_set); + if (ret < 0) + return ret; + } + + /* Lock onto this value until next refresh cycle */ + priv->duty_input[channel] = val; + + return 0; +} + +static int rog_ryujin_write(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, + long val) +{ + struct rog_ryujin_data *priv = dev_get_drvdata(dev); + int ret; + + switch (type) { + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_input: + if (val < 0 || val > 255) + return -EINVAL; + + ret = rog_ryujin_write_fixed_duty(priv, channel, val); + if (ret < 0) + return ret; + break; + default: + return -EOPNOTSUPP; + } + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + +static const struct hwmon_ops rog_ryujin_hwmon_ops = { + .is_visible = rog_ryujin_is_visible, + .read = rog_ryujin_read, + .read_string = rog_ryujin_read_string, + .write = rog_ryujin_write +}; + +static const struct hwmon_channel_info *rog_ryujin_info[] = { + HWMON_CHANNEL_INFO(temp, + HWMON_T_INPUT | HWMON_T_LABEL), + HWMON_CHANNEL_INFO(fan, + HWMON_F_INPUT | HWMON_F_LABEL, + HWMON_F_INPUT | HWMON_F_LABEL, + HWMON_F_INPUT | HWMON_F_LABEL, + HWMON_F_INPUT | HWMON_F_LABEL, + HWMON_F_INPUT | HWMON_F_LABEL, + HWMON_F_INPUT | HWMON_F_LABEL), + HWMON_CHANNEL_INFO(pwm, + HWMON_PWM_INPUT, + HWMON_PWM_INPUT, + HWMON_PWM_INPUT), + NULL +}; + +static const struct hwmon_chip_info rog_ryujin_chip_info = { + .ops = &rog_ryujin_hwmon_ops, + .info = rog_ryujin_info, +}; + +static int rog_ryujin_raw_event(struct hid_device *hdev, struct hid_report *report, u8 *data, + int size) +{ + struct rog_ryujin_data *priv = hid_get_drvdata(hdev); + + if (data[0] != RYUJIN_CMD_PREFIX) + return 0; + + if (data[1] == RYUJIN_GET_COOLER_STATUS_CMD_RESPONSE) { + /* Received coolant temp and speeds of pump and internal fan */ + priv->temp_input[0] = + data[RYUJIN_TEMP_SENSOR_1] * 1000 + data[RYUJIN_TEMP_SENSOR_2] * 100; + priv->speed_input[0] = get_unaligned_le16(data + RYUJIN_PUMP_SPEED); + priv->speed_input[1] = get_unaligned_le16(data + RYUJIN_INTERNAL_FAN_SPEED); + + if (!completion_done(&priv->cooler_status_received)) + complete_all(&priv->cooler_status_received); + } else if (data[1] == RYUJIN_GET_CONTROLLER_SPEED_CMD_RESPONSE) { + /* Received speeds of four fans attached to the controller */ + priv->speed_input[2] = get_unaligned_le16(data + RYUJIN_CONTROLLER_SPEED_1); + priv->speed_input[3] = get_unaligned_le16(data + RYUJIN_CONTROLLER_SPEED_2); + priv->speed_input[4] = get_unaligned_le16(data + RYUJIN_CONTROLLER_SPEED_3); + priv->speed_input[5] = get_unaligned_le16(data + RYUJIN_CONTROLLER_SPEED_4); + + if (!completion_done(&priv->controller_status_received)) + complete_all(&priv->controller_status_received); + } else if (data[1] == RYUJIN_GET_COOLER_DUTY_CMD_RESPONSE) { + /* Received report for pump and internal fan duties (in %) */ + if (data[RYUJIN_PUMP_DUTY] == 0 && data[RYUJIN_INTERNAL_FAN_DUTY] == 0) { + /* + * We received a report with zeroes for duty in both places. + * The device returns this as a confirmation that setting values + * is successful. If we initiated a write, mark it as complete. + */ + if (!completion_done(&priv->cooler_duty_set)) + complete_all(&priv->cooler_duty_set); + else if (!completion_done(&priv->cooler_duty_received)) + /* + * We didn't initiate a write, but received both zeroes. + * This means that either both duties are actually zero, + * or that we received a success report caused by userspace. + * We're expecting a report, so parse it. + */ + goto read_cooler_duty; + return 0; + } +read_cooler_duty: + priv->duty_input[0] = rog_ryujin_percent_to_pwm(data[RYUJIN_PUMP_DUTY]); + priv->duty_input[1] = rog_ryujin_percent_to_pwm(data[RYUJIN_INTERNAL_FAN_DUTY]); + + if (!completion_done(&priv->cooler_duty_received)) + complete_all(&priv->cooler_duty_received); + } else if (data[1] == RYUJIN_GET_CONTROLLER_DUTY_CMD_RESPONSE) { + /* Received report for controller duty for fans (in PWM) */ + if (data[RYUJIN_CONTROLLER_DUTY] == 0) { + /* + * We received a report with a zero for duty. The device returns this as + * a confirmation that setting the controller duty value was successful. + * If we initiated a write, mark it as complete. + */ + if (!completion_done(&priv->controller_duty_set)) + complete_all(&priv->controller_duty_set); + else if (!completion_done(&priv->controller_duty_received)) + /* + * We didn't initiate a write, but received a zero for duty. + * This means that either the duty is actually zero, or that + * we received a success report caused by userspace. + * We're expecting a report, so parse it. + */ + goto read_controller_duty; + return 0; + } +read_controller_duty: + priv->duty_input[2] = data[RYUJIN_CONTROLLER_DUTY]; + + if (!completion_done(&priv->controller_duty_received)) + complete_all(&priv->controller_duty_received); + } + + return 0; +} + +static int rog_ryujin_probe(struct hid_device *hdev, const struct hid_device_id *id) +{ + struct rog_ryujin_data *priv; + int ret; + + priv = devm_kzalloc(&hdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->hdev = hdev; + hid_set_drvdata(hdev, priv); + + /* + * Initialize priv->updated to STATUS_VALIDITY seconds in the past, making + * the initial empty data invalid for rog_ryujin_read() without the need for + * a special case there. + */ + priv->updated = jiffies - msecs_to_jiffies(STATUS_VALIDITY); + + ret = hid_parse(hdev); + if (ret) { + hid_err(hdev, "hid parse failed with %d\n", ret); + return ret; + } + + /* Enable hidraw so existing user-space tools can continue to work */ + ret = hid_hw_start(hdev, HID_CONNECT_HIDRAW); + if (ret) { + hid_err(hdev, "hid hw start failed with %d\n", ret); + return ret; + } + + ret = hid_hw_open(hdev); + if (ret) { + hid_err(hdev, "hid hw open failed with %d\n", ret); + goto fail_and_stop; + } + + priv->buffer = devm_kzalloc(&hdev->dev, MAX_REPORT_LENGTH, GFP_KERNEL); + if (!priv->buffer) { + ret = -ENOMEM; + goto fail_and_close; + } + + mutex_init(&priv->status_report_request_mutex); + mutex_init(&priv->buffer_lock); + spin_lock_init(&priv->status_report_request_lock); + init_completion(&priv->cooler_status_received); + init_completion(&priv->controller_status_received); + init_completion(&priv->cooler_duty_received); + init_completion(&priv->controller_duty_received); + init_completion(&priv->cooler_duty_set); + init_completion(&priv->controller_duty_set); + + priv->hwmon_dev = hwmon_device_register_with_info(&hdev->dev, "rog_ryujin", + priv, &rog_ryujin_chip_info, NULL); + if (IS_ERR(priv->hwmon_dev)) { + ret = PTR_ERR(priv->hwmon_dev); + hid_err(hdev, "hwmon registration failed with %d\n", ret); + goto fail_and_close; + } + + return 0; + +fail_and_close: + hid_hw_close(hdev); +fail_and_stop: + hid_hw_stop(hdev); + return ret; +} + +static void rog_ryujin_remove(struct hid_device *hdev) +{ + struct rog_ryujin_data *priv = hid_get_drvdata(hdev); + + hwmon_device_unregister(priv->hwmon_dev); + + hid_hw_close(hdev); + hid_hw_stop(hdev); +} + +static const struct hid_device_id rog_ryujin_table[] = { + { HID_USB_DEVICE(USB_VENDOR_ID_ASUS_ROG, USB_PRODUCT_ID_RYUJIN_AIO) }, + { } +}; + +MODULE_DEVICE_TABLE(hid, rog_ryujin_table); + +static struct hid_driver rog_ryujin_driver = { + .name = "rog_ryujin", + .id_table = rog_ryujin_table, + .probe = rog_ryujin_probe, + .remove = rog_ryujin_remove, + .raw_event = rog_ryujin_raw_event, +}; + +static int __init rog_ryujin_init(void) +{ + return hid_register_driver(&rog_ryujin_driver); +} + +static void __exit rog_ryujin_exit(void) +{ + hid_unregister_driver(&rog_ryujin_driver); +} + +/* When compiled into the kernel, initialize after the HID bus */ +late_initcall(rog_ryujin_init); +module_exit(rog_ryujin_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Aleksa Savic "); +MODULE_DESCRIPTION("Hwmon driver for Asus ROG Ryujin II 360 AIO cooler"); diff --git a/drivers/hwmon/axi-fan-control.c b/drivers/hwmon/axi-fan-control.c index 19b9bf3d75ef..35c862eb158b 100644 --- a/drivers/hwmon/axi-fan-control.c +++ b/drivers/hwmon/axi-fan-control.c @@ -13,8 +13,9 @@ #include #include #include -#include +#include #include +#include /* register map */ #define ADI_REG_RSTN 0x0080 @@ -83,7 +84,7 @@ static ssize_t axi_fan_control_show(struct device *dev, struct device_attribute temp = DIV_ROUND_CLOSEST_ULL(temp * 509314ULL, 65535) - 280230; - return sprintf(buf, "%u\n", temp); + return sysfs_emit(buf, "%u\n", temp); } static ssize_t axi_fan_control_store(struct device *dev, struct device_attribute *da, @@ -368,12 +369,12 @@ static irqreturn_t axi_fan_control_irq_handler(int irq, void *data) } static int axi_fan_control_init(struct axi_fan_control_data *ctl, - const struct device_node *np) + const struct device *dev) { int ret; /* get fan pulses per revolution */ - ret = of_property_read_u32(np, "pulses-per-revolution", &ctl->ppr); + ret = device_property_read_u32(dev, "pulses-per-revolution", &ctl->ppr); if (ret) return ret; @@ -443,25 +444,16 @@ static struct attribute *axi_fan_control_attrs[] = { }; ATTRIBUTE_GROUPS(axi_fan_control); -static const u32 version_1_0_0 = ADI_AXI_PCORE_VER(1, 0, 'a'); - -static const struct of_device_id axi_fan_control_of_match[] = { - { .compatible = "adi,axi-fan-control-1.00.a", - .data = (void *)&version_1_0_0}, - {}, -}; -MODULE_DEVICE_TABLE(of, axi_fan_control_of_match); - static int axi_fan_control_probe(struct platform_device *pdev) { struct axi_fan_control_data *ctl; struct clk *clk; - const struct of_device_id *id; + const unsigned int *id; const char *name = "axi_fan_control"; u32 version; int ret; - id = of_match_node(axi_fan_control_of_match, pdev->dev.of_node); + id = device_get_match_data(&pdev->dev); if (!id) return -EINVAL; @@ -474,10 +466,9 @@ static int axi_fan_control_probe(struct platform_device *pdev) return PTR_ERR(ctl->base); clk = devm_clk_get_enabled(&pdev->dev, NULL); - if (IS_ERR(clk)) { - dev_err(&pdev->dev, "clk_get failed with %ld\n", PTR_ERR(clk)); - return PTR_ERR(clk); - } + if (IS_ERR(clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(clk), + "clk_get failed\n"); ctl->clk_rate = clk_get_rate(clk); if (!ctl->clk_rate) @@ -485,22 +476,20 @@ static int axi_fan_control_probe(struct platform_device *pdev) version = axi_ioread(ADI_AXI_REG_VERSION, ctl); if (ADI_AXI_PCORE_VER_MAJOR(version) != - ADI_AXI_PCORE_VER_MAJOR((*(u32 *)id->data))) { - dev_err(&pdev->dev, "Major version mismatch. Expected %d.%.2d.%c, Reported %d.%.2d.%c\n", - ADI_AXI_PCORE_VER_MAJOR((*(u32 *)id->data)), - ADI_AXI_PCORE_VER_MINOR((*(u32 *)id->data)), - ADI_AXI_PCORE_VER_PATCH((*(u32 *)id->data)), - ADI_AXI_PCORE_VER_MAJOR(version), - ADI_AXI_PCORE_VER_MINOR(version), - ADI_AXI_PCORE_VER_PATCH(version)); - return -ENODEV; - } + ADI_AXI_PCORE_VER_MAJOR((*id))) + return dev_err_probe(&pdev->dev, -ENODEV, + "Major version mismatch. Expected %d.%.2d.%c, Reported %d.%.2d.%c\n", + ADI_AXI_PCORE_VER_MAJOR(*id), + ADI_AXI_PCORE_VER_MINOR(*id), + ADI_AXI_PCORE_VER_PATCH(*id), + ADI_AXI_PCORE_VER_MAJOR(version), + ADI_AXI_PCORE_VER_MINOR(version), + ADI_AXI_PCORE_VER_PATCH(version)); - ret = axi_fan_control_init(ctl, pdev->dev.of_node); - if (ret) { - dev_err(&pdev->dev, "Failed to initialize device\n"); - return ret; - } + ret = axi_fan_control_init(ctl, &pdev->dev); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Failed to initialize device\n"); ctl->hdev = devm_hwmon_device_register_with_info(&pdev->dev, name, @@ -519,14 +508,22 @@ static int axi_fan_control_probe(struct platform_device *pdev) axi_fan_control_irq_handler, IRQF_ONESHOT | IRQF_TRIGGER_HIGH, pdev->driver_override, ctl); - if (ret) { - dev_err(&pdev->dev, "failed to request an irq, %d", ret); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "failed to request an irq\n"); return 0; } +static const u32 version_1_0_0 = ADI_AXI_PCORE_VER(1, 0, 'a'); + +static const struct of_device_id axi_fan_control_of_match[] = { + { .compatible = "adi,axi-fan-control-1.00.a", + .data = (void *)&version_1_0_0}, + {}, +}; +MODULE_DEVICE_TABLE(of, axi_fan_control_of_match); + static struct platform_driver axi_fan_control_driver = { .driver = { .name = "axi_fan_control_driver", diff --git a/drivers/hwmon/chipcap2.c b/drivers/hwmon/chipcap2.c new file mode 100644 index 000000000000..6ccceae21f70 --- /dev/null +++ b/drivers/hwmon/chipcap2.c @@ -0,0 +1,822 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * cc2.c - Support for the Amphenol ChipCap 2 relative humidity, temperature sensor + * + * Part numbers supported: + * CC2D23, CC2D23S, CC2D25, CC2D25S, CC2D33, CC2D33S, CC2D35, CC2D35S + * + * Author: Javier Carrasco + * + * Datasheet and application notes: + * https://www.amphenol-sensors.com/en/telaire/humidity/527-humidity-sensors/3095-chipcap-2 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CC2_START_CM 0xA0 +#define CC2_START_NOM 0x80 +#define CC2_R_ALARM_H_ON 0x18 +#define CC2_R_ALARM_H_OFF 0x19 +#define CC2_R_ALARM_L_ON 0x1A +#define CC2_R_ALARM_L_OFF 0x1B +#define CC2_RW_OFFSET 0x40 +#define CC2_W_ALARM_H_ON (CC2_R_ALARM_H_ON + CC2_RW_OFFSET) +#define CC2_W_ALARM_H_OFF (CC2_R_ALARM_H_OFF + CC2_RW_OFFSET) +#define CC2_W_ALARM_L_ON (CC2_R_ALARM_L_ON + CC2_RW_OFFSET) +#define CC2_W_ALARM_L_OFF (CC2_R_ALARM_L_OFF + CC2_RW_OFFSET) + +#define CC2_STATUS_FIELD GENMASK(7, 6) +#define CC2_STATUS_VALID_DATA 0x00 +#define CC2_STATUS_STALE_DATA 0x01 +#define CC2_STATUS_CMD_MODE 0x02 + +#define CC2_RESPONSE_FIELD GENMASK(1, 0) +#define CC2_RESPONSE_BUSY 0x00 +#define CC2_RESPONSE_ACK 0x01 +#define CC2_RESPONSE_NACK 0x02 + +#define CC2_ERR_CORR_EEPROM BIT(2) +#define CC2_ERR_UNCORR_EEPROM BIT(3) +#define CC2_ERR_RAM_PARITY BIT(4) +#define CC2_ERR_CONFIG_LOAD BIT(5) + +#define CC2_EEPROM_SIZE 10 +#define CC2_EEPROM_DATA_LEN 3 +#define CC2_MEASUREMENT_DATA_LEN 4 + +#define CC2_RH_DATA_FIELD GENMASK(13, 0) + +/* ensure clean off -> on transitions */ +#define CC2_POWER_CYCLE_MS 80 + +#define CC2_STARTUP_TO_DATA_MS 55 +#define CC2_RESP_START_CM_US 100 +#define CC2_RESP_EEPROM_R_US 100 +#define CC2_RESP_EEPROM_W_MS 12 +#define CC2_STARTUP_TIME_US 1250 + +#define CC2_RH_MAX (100 * 1000U) + +#define CC2_CM_RETRIES 5 + +struct cc2_rh_alarm_info { + bool low_alarm; + bool high_alarm; + bool low_alarm_visible; + bool high_alarm_visible; +}; + +struct cc2_data { + struct cc2_rh_alarm_info rh_alarm; + struct completion complete; + struct device *hwmon; + struct i2c_client *client; + struct mutex dev_access_lock; /* device access lock */ + struct regulator *regulator; + const char *name; + int irq_ready; + int irq_low; + int irq_high; + bool process_irqs; +}; + +enum cc2_chan_addr { + CC2_CHAN_TEMP = 0, + CC2_CHAN_HUMIDITY, +}; + +/* %RH as a per cent mille from a register value */ +static long cc2_rh_convert(u16 data) +{ + unsigned long tmp = (data & CC2_RH_DATA_FIELD) * CC2_RH_MAX; + + return tmp / ((1 << 14) - 1); +} + +/* convert %RH to a register value */ +static u16 cc2_rh_to_reg(long data) +{ + return data * ((1 << 14) - 1) / CC2_RH_MAX; +} + +/* temperature in milli degrees celsius from a register value */ +static long cc2_temp_convert(u16 data) +{ + unsigned long tmp = ((data >> 2) * 165 * 1000U) / ((1 << 14) - 1); + + return tmp - 40 * 1000U; +} + +static int cc2_enable(struct cc2_data *data) +{ + int ret; + + /* exclusive regulator, check in case a disable failed */ + if (regulator_is_enabled(data->regulator)) + return 0; + + /* clear any pending completion */ + try_wait_for_completion(&data->complete); + + ret = regulator_enable(data->regulator); + if (ret < 0) + return ret; + + usleep_range(CC2_STARTUP_TIME_US, CC2_STARTUP_TIME_US + 125); + + data->process_irqs = true; + + return 0; +} + +static void cc2_disable(struct cc2_data *data) +{ + int err; + + /* ignore alarms triggered by voltage toggling when powering up */ + data->process_irqs = false; + + /* exclusive regulator, check in case an enable failed */ + if (regulator_is_enabled(data->regulator)) { + err = regulator_disable(data->regulator); + if (err) + dev_dbg(&data->client->dev, "Failed to disable device"); + } +} + +static int cc2_cmd_response_diagnostic(struct device *dev, u8 status) +{ + int resp; + + if (FIELD_GET(CC2_STATUS_FIELD, status) != CC2_STATUS_CMD_MODE) { + dev_dbg(dev, "Command sent out of command window\n"); + return -ETIMEDOUT; + } + + resp = FIELD_GET(CC2_RESPONSE_FIELD, status); + switch (resp) { + case CC2_RESPONSE_ACK: + return 0; + case CC2_RESPONSE_BUSY: + return -EBUSY; + case CC2_RESPONSE_NACK: + if (resp & CC2_ERR_CORR_EEPROM) + dev_dbg(dev, "Command failed: corrected EEPROM\n"); + if (resp & CC2_ERR_UNCORR_EEPROM) + dev_dbg(dev, "Command failed: uncorrected EEPROM\n"); + if (resp & CC2_ERR_RAM_PARITY) + dev_dbg(dev, "Command failed: RAM parity\n"); + if (resp & CC2_ERR_RAM_PARITY) + dev_dbg(dev, "Command failed: configuration error\n"); + return -ENODATA; + default: + dev_dbg(dev, "Unknown command reply\n"); + return -EINVAL; + } +} + +static int cc2_read_command_status(struct i2c_client *client) +{ + u8 status; + int ret; + + ret = i2c_master_recv(client, &status, 1); + if (ret != 1) { + ret = ret < 0 ? ret : -EIO; + return ret; + } + + return cc2_cmd_response_diagnostic(&client->dev, status); +} + +/* + * The command mode is only accessible after sending the START_CM command in the + * first 10 ms after power-up. Only in case the command window is missed, + * CC2_CM_RETRIES retries are attempted before giving up and returning an error. + */ +static int cc2_command_mode_start(struct cc2_data *data) +{ + unsigned long timeout; + int i, ret; + + for (i = 0; i < CC2_CM_RETRIES; i++) { + ret = cc2_enable(data); + if (ret < 0) + return ret; + + ret = i2c_smbus_write_word_data(data->client, CC2_START_CM, 0); + if (ret < 0) + return ret; + + if (data->irq_ready > 0) { + timeout = usecs_to_jiffies(2 * CC2_RESP_START_CM_US); + ret = wait_for_completion_timeout(&data->complete, + timeout); + if (!ret) + return -ETIMEDOUT; + } else { + usleep_range(CC2_RESP_START_CM_US, + 2 * CC2_RESP_START_CM_US); + } + ret = cc2_read_command_status(data->client); + if (ret != -ETIMEDOUT || i == CC2_CM_RETRIES) + break; + + /* command window missed, prepare for a retry */ + cc2_disable(data); + msleep(CC2_POWER_CYCLE_MS); + } + + return ret; +} + +/* Sending a Start_NOM command finishes the command mode immediately with no + * reply and the device enters normal operation mode + */ +static int cc2_command_mode_finish(struct cc2_data *data) +{ + int ret; + + ret = i2c_smbus_write_word_data(data->client, CC2_START_NOM, 0); + if (ret < 0) + return ret; + + return 0; +} + +static int cc2_write_reg(struct cc2_data *data, u8 reg, u16 val) +{ + unsigned long timeout; + int ret; + + ret = cc2_command_mode_start(data); + if (ret < 0) + goto disable; + + cpu_to_be16s(&val); + ret = i2c_smbus_write_word_data(data->client, reg, val); + if (ret < 0) + goto disable; + + if (data->irq_ready > 0) { + timeout = msecs_to_jiffies(2 * CC2_RESP_EEPROM_W_MS); + ret = wait_for_completion_timeout(&data->complete, timeout); + if (!ret) { + ret = -ETIMEDOUT; + goto disable; + } + } else { + msleep(CC2_RESP_EEPROM_W_MS); + } + + ret = cc2_read_command_status(data->client); + +disable: + cc2_disable(data); + + return ret; +} + +static int cc2_read_reg(struct cc2_data *data, u8 reg, u16 *val) +{ + u8 buf[CC2_EEPROM_DATA_LEN]; + unsigned long timeout; + int ret; + + ret = cc2_command_mode_start(data); + if (ret < 0) + return ret; + + ret = i2c_smbus_write_word_data(data->client, reg, 0); + if (ret < 0) + return ret; + + if (data->irq_ready > 0) { + timeout = usecs_to_jiffies(2 * CC2_RESP_EEPROM_R_US); + ret = wait_for_completion_timeout(&data->complete, timeout); + if (!ret) + return -ETIMEDOUT; + + } else { + usleep_range(CC2_RESP_EEPROM_R_US, CC2_RESP_EEPROM_R_US + 10); + } + ret = i2c_master_recv(data->client, buf, CC2_EEPROM_DATA_LEN); + if (ret != CC2_EEPROM_DATA_LEN) + return ret < 0 ? ret : -EIO; + + *val = be16_to_cpup((__be16 *)&buf[1]); + + return cc2_read_command_status(data->client); +} + +static int cc2_get_reg_val(struct cc2_data *data, u8 reg, long *val) +{ + u16 reg_val; + int ret; + + ret = cc2_read_reg(data, reg, ®_val); + if (!ret) + *val = cc2_rh_convert(reg_val); + + cc2_disable(data); + + return ret; +} + +static int cc2_data_fetch(struct i2c_client *client, + enum hwmon_sensor_types type, long *val) +{ + u8 data[CC2_MEASUREMENT_DATA_LEN]; + u8 status; + int ret; + + ret = i2c_master_recv(client, data, CC2_MEASUREMENT_DATA_LEN); + if (ret != CC2_MEASUREMENT_DATA_LEN) { + ret = ret < 0 ? ret : -EIO; + return ret; + } + status = FIELD_GET(CC2_STATUS_FIELD, data[0]); + if (status == CC2_STATUS_STALE_DATA) + return -EBUSY; + + if (status != CC2_STATUS_VALID_DATA) + return -EIO; + + switch (type) { + case hwmon_humidity: + *val = cc2_rh_convert(be16_to_cpup((__be16 *)&data[0])); + break; + case hwmon_temp: + *val = cc2_temp_convert(be16_to_cpup((__be16 *)&data[2])); + break; + default: + return -EINVAL; + } + + return 0; +} + +static int cc2_read_measurement(struct cc2_data *data, + enum hwmon_sensor_types type, long *val) +{ + unsigned long timeout; + int ret; + + if (data->irq_ready > 0) { + timeout = msecs_to_jiffies(CC2_STARTUP_TO_DATA_MS * 2); + ret = wait_for_completion_timeout(&data->complete, timeout); + if (!ret) + return -ETIMEDOUT; + + } else { + msleep(CC2_STARTUP_TO_DATA_MS); + } + + ret = cc2_data_fetch(data->client, type, val); + + return ret; +} + +/* + * A measurement requires enabling the device, waiting for the automatic + * measurement to finish, reading the measurement data and disabling the device + * again. + */ +static int cc2_measurement(struct cc2_data *data, enum hwmon_sensor_types type, + long *val) +{ + int ret; + + ret = cc2_enable(data); + if (ret) + return ret; + + ret = cc2_read_measurement(data, type, val); + + cc2_disable(data); + + return ret; +} + +/* + * In order to check alarm status, the corresponding ALARM_OFF (hysteresis) + * register must be read and a new measurement must be carried out to trigger + * the alarm signals. Given that the device carries out a measurement after + * exiting the command mode, there is no need to force two power-up sequences. + * Instead, a NOM command is sent and the device is disabled after the + * measurement is read. + */ +static int cc2_read_hyst_and_measure(struct cc2_data *data, u8 reg, + long *hyst, long *measurement) +{ + u16 reg_val; + int ret; + + ret = cc2_read_reg(data, reg, ®_val); + if (ret) + goto disable; + + *hyst = cc2_rh_convert(reg_val); + + ret = cc2_command_mode_finish(data); + if (ret) + goto disable; + + ret = cc2_read_measurement(data, hwmon_humidity, measurement); + +disable: + cc2_disable(data); + + return ret; +} + +static umode_t cc2_is_visible(const void *data, enum hwmon_sensor_types type, + u32 attr, int channel) +{ + const struct cc2_data *cc2 = data; + + switch (type) { + case hwmon_humidity: + switch (attr) { + case hwmon_humidity_input: + return 0444; + case hwmon_humidity_min_alarm: + return cc2->rh_alarm.low_alarm_visible ? 0444 : 0; + case hwmon_humidity_max_alarm: + return cc2->rh_alarm.high_alarm_visible ? 0444 : 0; + case hwmon_humidity_min: + case hwmon_humidity_min_hyst: + return cc2->rh_alarm.low_alarm_visible ? 0644 : 0; + case hwmon_humidity_max: + case hwmon_humidity_max_hyst: + return cc2->rh_alarm.high_alarm_visible ? 0644 : 0; + default: + return 0; + } + case hwmon_temp: + switch (attr) { + case hwmon_temp_input: + return 0444; + default: + return 0; + } + default: + break; + } + + return 0; +} + +static irqreturn_t cc2_ready_interrupt(int irq, void *data) +{ + struct cc2_data *cc2 = data; + + if (cc2->process_irqs) + complete(&cc2->complete); + + return IRQ_HANDLED; +} + +static irqreturn_t cc2_low_interrupt(int irq, void *data) +{ + struct cc2_data *cc2 = data; + + if (cc2->process_irqs) { + hwmon_notify_event(cc2->hwmon, hwmon_humidity, + hwmon_humidity_min_alarm, CC2_CHAN_HUMIDITY); + cc2->rh_alarm.low_alarm = true; + } + + return IRQ_HANDLED; +} + +static irqreturn_t cc2_high_interrupt(int irq, void *data) +{ + struct cc2_data *cc2 = data; + + if (cc2->process_irqs) { + hwmon_notify_event(cc2->hwmon, hwmon_humidity, + hwmon_humidity_max_alarm, CC2_CHAN_HUMIDITY); + cc2->rh_alarm.high_alarm = true; + } + + return IRQ_HANDLED; +} + +static int cc2_humidity_min_alarm_status(struct cc2_data *data, long *val) +{ + long measurement, min_hyst; + int ret; + + ret = cc2_read_hyst_and_measure(data, CC2_R_ALARM_L_OFF, &min_hyst, + &measurement); + if (ret < 0) + return ret; + + if (data->rh_alarm.low_alarm) { + *val = (measurement < min_hyst) ? 1 : 0; + data->rh_alarm.low_alarm = *val; + } else { + *val = 0; + } + + return 0; +} + +static int cc2_humidity_max_alarm_status(struct cc2_data *data, long *val) +{ + long measurement, max_hyst; + int ret; + + ret = cc2_read_hyst_and_measure(data, CC2_R_ALARM_H_OFF, &max_hyst, + &measurement); + if (ret < 0) + return ret; + + if (data->rh_alarm.high_alarm) { + *val = (measurement > max_hyst) ? 1 : 0; + data->rh_alarm.high_alarm = *val; + } else { + *val = 0; + } + + return 0; +} + +static int cc2_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, + int channel, long *val) +{ + struct cc2_data *data = dev_get_drvdata(dev); + int ret = 0; + + mutex_lock(&data->dev_access_lock); + + switch (type) { + case hwmon_temp: + ret = cc2_measurement(data, type, val); + break; + case hwmon_humidity: + switch (attr) { + case hwmon_humidity_input: + ret = cc2_measurement(data, type, val); + break; + case hwmon_humidity_min: + ret = cc2_get_reg_val(data, CC2_R_ALARM_L_ON, val); + break; + case hwmon_humidity_min_hyst: + ret = cc2_get_reg_val(data, CC2_R_ALARM_L_OFF, val); + break; + case hwmon_humidity_max: + ret = cc2_get_reg_val(data, CC2_R_ALARM_H_ON, val); + break; + case hwmon_humidity_max_hyst: + ret = cc2_get_reg_val(data, CC2_R_ALARM_H_OFF, val); + break; + case hwmon_humidity_min_alarm: + ret = cc2_humidity_min_alarm_status(data, val); + break; + case hwmon_humidity_max_alarm: + ret = cc2_humidity_max_alarm_status(data, val); + break; + default: + ret = -EOPNOTSUPP; + } + break; + default: + ret = -EOPNOTSUPP; + } + + mutex_unlock(&data->dev_access_lock); + + return ret; +} + +static int cc2_write(struct device *dev, enum hwmon_sensor_types type, u32 attr, + int channel, long val) +{ + struct cc2_data *data = dev_get_drvdata(dev); + int ret; + u16 arg; + u8 cmd; + + if (type != hwmon_humidity) + return -EOPNOTSUPP; + + if (val < 0 || val > CC2_RH_MAX) + return -EINVAL; + + mutex_lock(&data->dev_access_lock); + + switch (attr) { + case hwmon_humidity_min: + cmd = CC2_W_ALARM_L_ON; + arg = cc2_rh_to_reg(val); + ret = cc2_write_reg(data, cmd, arg); + break; + + case hwmon_humidity_min_hyst: + cmd = CC2_W_ALARM_L_OFF; + arg = cc2_rh_to_reg(val); + ret = cc2_write_reg(data, cmd, arg); + break; + + case hwmon_humidity_max: + cmd = CC2_W_ALARM_H_ON; + arg = cc2_rh_to_reg(val); + ret = cc2_write_reg(data, cmd, arg); + break; + + case hwmon_humidity_max_hyst: + cmd = CC2_W_ALARM_H_OFF; + arg = cc2_rh_to_reg(val); + ret = cc2_write_reg(data, cmd, arg); + break; + + default: + ret = -EOPNOTSUPP; + break; + } + + mutex_unlock(&data->dev_access_lock); + + return ret; +} + +static int cc2_request_ready_irq(struct cc2_data *data, struct device *dev) +{ + int ret = 0; + + data->irq_ready = fwnode_irq_get_byname(dev_fwnode(dev), "ready"); + if (data->irq_ready > 0) { + init_completion(&data->complete); + ret = devm_request_threaded_irq(dev, data->irq_ready, NULL, + cc2_ready_interrupt, + IRQF_ONESHOT | + IRQF_TRIGGER_RISING, + dev_name(dev), data); + } + + return ret; +} + +static int cc2_request_alarm_irqs(struct cc2_data *data, struct device *dev) +{ + int ret = 0; + + data->irq_low = fwnode_irq_get_byname(dev_fwnode(dev), "low"); + if (data->irq_low > 0) { + ret = devm_request_threaded_irq(dev, data->irq_low, NULL, + cc2_low_interrupt, + IRQF_ONESHOT | + IRQF_TRIGGER_RISING, + dev_name(dev), data); + if (ret) + return ret; + + data->rh_alarm.low_alarm_visible = true; + } + + data->irq_high = fwnode_irq_get_byname(dev_fwnode(dev), "high"); + if (data->irq_high > 0) { + ret = devm_request_threaded_irq(dev, data->irq_high, NULL, + cc2_high_interrupt, + IRQF_ONESHOT | + IRQF_TRIGGER_RISING, + dev_name(dev), data); + if (ret) + return ret; + + data->rh_alarm.high_alarm_visible = true; + } + + return ret; +} + +static const struct hwmon_channel_info *cc2_info[] = { + HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT), + HWMON_CHANNEL_INFO(humidity, HWMON_H_INPUT | HWMON_H_MIN | HWMON_H_MAX | + HWMON_H_MIN_HYST | HWMON_H_MAX_HYST | + HWMON_H_MIN_ALARM | HWMON_H_MAX_ALARM), + NULL +}; + +static const struct hwmon_ops cc2_hwmon_ops = { + .is_visible = cc2_is_visible, + .read = cc2_read, + .write = cc2_write, +}; + +static const struct hwmon_chip_info cc2_chip_info = { + .ops = &cc2_hwmon_ops, + .info = cc2_info, +}; + +static int cc2_probe(struct i2c_client *client) +{ + struct cc2_data *data; + struct device *dev = &client->dev; + int ret; + + if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) + return -EOPNOTSUPP; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + i2c_set_clientdata(client, data); + + mutex_init(&data->dev_access_lock); + + data->client = client; + + data->regulator = devm_regulator_get_exclusive(dev, "vdd"); + if (IS_ERR(data->regulator)) { + dev_err_probe(dev, PTR_ERR(data->regulator), + "Failed to get regulator\n"); + return PTR_ERR(data->regulator); + } + + ret = cc2_request_ready_irq(data, dev); + if (ret) { + dev_err_probe(dev, ret, "Failed to request ready irq\n"); + return ret; + } + + ret = cc2_request_alarm_irqs(data, dev); + if (ret) { + dev_err_probe(dev, ret, "Failed to request alarm irqs\n"); + goto disable; + } + + data->hwmon = devm_hwmon_device_register_with_info(dev, client->name, + data, &cc2_chip_info, + NULL); + if (IS_ERR(data->hwmon)) { + dev_err_probe(dev, PTR_ERR(data->hwmon), + "Failed to register hwmon device\n"); + ret = PTR_ERR(data->hwmon); + } + +disable: + cc2_disable(data); + + return ret; +} + +static void cc2_remove(struct i2c_client *client) +{ + struct cc2_data *data = i2c_get_clientdata(client); + + cc2_disable(data); +} + +static const struct i2c_device_id cc2_id[] = { + { "cc2d23" }, + { "cc2d23s" }, + { "cc2d25" }, + { "cc2d25s" }, + { "cc2d33" }, + { "cc2d33s" }, + { "cc2d35" }, + { "cc2d35s" }, + { } +}; +MODULE_DEVICE_TABLE(i2c, cc2_id); + +static const struct of_device_id cc2_of_match[] = { + { .compatible = "amphenol,cc2d23" }, + { .compatible = "amphenol,cc2d23s" }, + { .compatible = "amphenol,cc2d25" }, + { .compatible = "amphenol,cc2d25s" }, + { .compatible = "amphenol,cc2d33" }, + { .compatible = "amphenol,cc2d33s" }, + { .compatible = "amphenol,cc2d35" }, + { .compatible = "amphenol,cc2d35s" }, + { }, +}; +MODULE_DEVICE_TABLE(of, cc2_of_match); + +static struct i2c_driver cc2_driver = { + .driver = { + .name = "cc2d23", + .of_match_table = cc2_of_match, + }, + .probe = cc2_probe, + .remove = cc2_remove, + .id_table = cc2_id, +}; +module_i2c_driver(cc2_driver); + +MODULE_AUTHOR("Javier Carrasco "); +MODULE_DESCRIPTION("Amphenol ChipCap 2 humidity and temperature sensor driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c index b0991dde2e59..616bd1a5b864 100644 --- a/drivers/hwmon/coretemp.c +++ b/drivers/hwmon/coretemp.c @@ -39,13 +39,18 @@ static int force_tjmax; module_param_named(tjmax, force_tjmax, int, 0444); MODULE_PARM_DESC(tjmax, "TjMax value in degrees Celsius"); -#define PKG_SYSFS_ATTR_NO 1 /* Sysfs attribute for package temp */ -#define BASE_SYSFS_ATTR_NO 2 /* Sysfs Base attr no for coretemp */ #define NUM_REAL_CORES 512 /* Number of Real cores per cpu */ #define CORETEMP_NAME_LENGTH 28 /* String Length of attrs */ -#define MAX_CORE_ATTRS 4 /* Maximum no of basic attrs */ -#define TOTAL_ATTRS (MAX_CORE_ATTRS + 1) -#define MAX_CORE_DATA (NUM_REAL_CORES + BASE_SYSFS_ATTR_NO) + +enum coretemp_attr_index { + ATTR_LABEL, + ATTR_CRIT_ALARM, + ATTR_TEMP, + ATTR_TJMAX, + ATTR_TTARGET, + MAX_CORE_ATTRS = ATTR_TJMAX + 1, /* Maximum no of basic attrs */ + TOTAL_ATTRS = ATTR_TTARGET + 1 /* Maximum no of possible attrs */ +}; #ifdef CONFIG_SMP #define for_each_sibling(i, cpu) \ @@ -65,19 +70,17 @@ MODULE_PARM_DESC(tjmax, "TjMax value in degrees Celsius"); * @status_reg: One of IA32_THERM_STATUS or IA32_PACKAGE_THERM_STATUS, * from where the temperature values should be read. * @attr_size: Total number of pre-core attrs displayed in the sysfs. - * @is_pkg_data: If this is 1, the temp_data holds pkgtemp data. - * Otherwise, temp_data holds coretemp data. */ struct temp_data { int temp; int tjmax; unsigned long last_updated; unsigned int cpu; + int index; u32 cpu_core_id; u32 status_reg; int attr_size; - bool is_pkg_data; - struct sensor_device_attribute sd_attrs[TOTAL_ATTRS]; + struct device_attribute sd_attrs[TOTAL_ATTRS]; char attr_name[TOTAL_ATTRS][CORETEMP_NAME_LENGTH]; struct attribute *attrs[TOTAL_ATTRS + 1]; struct attribute_group attr_group; @@ -88,10 +91,11 @@ struct temp_data { struct platform_data { struct device *hwmon_dev; u16 pkg_id; - u16 cpu_map[NUM_REAL_CORES]; + int nr_cores; struct ida ida; struct cpumask cpumask; - struct temp_data *core_data[MAX_CORE_DATA]; + struct temp_data *pkg_data; + struct temp_data **core_data; struct device_attribute name_attr; }; @@ -143,6 +147,11 @@ static const struct tjmax_model tjmax_model_table[] = { */ }; +static bool is_pkg_temp_data(struct temp_data *tdata) +{ + return tdata->index < 0; +} + static int adjust_tjmax(struct cpuinfo_x86 *c, u32 id, struct device *dev) { /* The 100C is default for both mobile and non mobile CPUs */ @@ -332,11 +341,10 @@ static struct platform_device **zone_devices; static ssize_t show_label(struct device *dev, struct device_attribute *devattr, char *buf) { - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); struct platform_data *pdata = dev_get_drvdata(dev); - struct temp_data *tdata = pdata->core_data[attr->index]; + struct temp_data *tdata = container_of(devattr, struct temp_data, sd_attrs[ATTR_LABEL]); - if (tdata->is_pkg_data) + if (is_pkg_temp_data(tdata)) return sprintf(buf, "Package id %u\n", pdata->pkg_id); return sprintf(buf, "Core %u\n", tdata->cpu_core_id); @@ -346,9 +354,8 @@ static ssize_t show_crit_alarm(struct device *dev, struct device_attribute *devattr, char *buf) { u32 eax, edx; - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct platform_data *pdata = dev_get_drvdata(dev); - struct temp_data *tdata = pdata->core_data[attr->index]; + struct temp_data *tdata = container_of(devattr, struct temp_data, + sd_attrs[ATTR_CRIT_ALARM]); mutex_lock(&tdata->update_lock); rdmsr_on_cpu(tdata->cpu, tdata->status_reg, &eax, &edx); @@ -360,9 +367,7 @@ static ssize_t show_crit_alarm(struct device *dev, static ssize_t show_tjmax(struct device *dev, struct device_attribute *devattr, char *buf) { - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct platform_data *pdata = dev_get_drvdata(dev); - struct temp_data *tdata = pdata->core_data[attr->index]; + struct temp_data *tdata = container_of(devattr, struct temp_data, sd_attrs[ATTR_TJMAX]); int tjmax; mutex_lock(&tdata->update_lock); @@ -375,9 +380,7 @@ static ssize_t show_tjmax(struct device *dev, static ssize_t show_ttarget(struct device *dev, struct device_attribute *devattr, char *buf) { - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct platform_data *pdata = dev_get_drvdata(dev); - struct temp_data *tdata = pdata->core_data[attr->index]; + struct temp_data *tdata = container_of(devattr, struct temp_data, sd_attrs[ATTR_TTARGET]); int ttarget; mutex_lock(&tdata->update_lock); @@ -393,9 +396,7 @@ static ssize_t show_temp(struct device *dev, struct device_attribute *devattr, char *buf) { u32 eax, edx; - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct platform_data *pdata = dev_get_drvdata(dev); - struct temp_data *tdata = pdata->core_data[attr->index]; + struct temp_data *tdata = container_of(devattr, struct temp_data, sd_attrs[ATTR_TEMP]); int tjmax; mutex_lock(&tdata->update_lock); @@ -418,8 +419,7 @@ static ssize_t show_temp(struct device *dev, return sprintf(buf, "%d\n", tdata->temp); } -static int create_core_attrs(struct temp_data *tdata, struct device *dev, - int index) +static int create_core_attrs(struct temp_data *tdata, struct device *dev) { int i; static ssize_t (*const rd_ptr[TOTAL_ATTRS]) (struct device *dev, @@ -436,16 +436,15 @@ static int create_core_attrs(struct temp_data *tdata, struct device *dev, * The attr number is always core id + 2 * The Pkgtemp will always show up as temp1_*, if available */ - int attr_no = tdata->is_pkg_data ? 1 : tdata->cpu_core_id + 2; + int attr_no = is_pkg_temp_data(tdata) ? 1 : tdata->cpu_core_id + 2; snprintf(tdata->attr_name[i], CORETEMP_NAME_LENGTH, "temp%d_%s", attr_no, suffixes[i]); - sysfs_attr_init(&tdata->sd_attrs[i].dev_attr.attr); - tdata->sd_attrs[i].dev_attr.attr.name = tdata->attr_name[i]; - tdata->sd_attrs[i].dev_attr.attr.mode = 0444; - tdata->sd_attrs[i].dev_attr.show = rd_ptr[i]; - tdata->sd_attrs[i].index = index; - tdata->attrs[i] = &tdata->sd_attrs[i].dev_attr.attr; + sysfs_attr_init(&tdata->sd_attrs[i].attr); + tdata->sd_attrs[i].attr.name = tdata->attr_name[i]; + tdata->sd_attrs[i].attr.mode = 0444; + tdata->sd_attrs[i].show = rd_ptr[i]; + tdata->attrs[i] = &tdata->sd_attrs[i].attr; } tdata->attr_group.attrs = tdata->attrs; return sysfs_create_group(&dev->kobj, &tdata->attr_group); @@ -477,17 +476,44 @@ static struct platform_device *coretemp_get_pdev(unsigned int cpu) return NULL; } -static struct temp_data *init_temp_data(unsigned int cpu, int pkg_flag) +static struct temp_data * +init_temp_data(struct platform_data *pdata, unsigned int cpu, int pkg_flag) { struct temp_data *tdata; + if (!pdata->core_data) { + /* + * TODO: + * The information of actual possible cores in a package is broken for now. + * Will replace hardcoded NUM_REAL_CORES with actual per package core count + * when this information becomes available. + */ + pdata->nr_cores = NUM_REAL_CORES; + pdata->core_data = kcalloc(pdata->nr_cores, sizeof(struct temp_data *), + GFP_KERNEL); + if (!pdata->core_data) + return NULL; + } + tdata = kzalloc(sizeof(struct temp_data), GFP_KERNEL); if (!tdata) return NULL; + if (pkg_flag) { + pdata->pkg_data = tdata; + /* Use tdata->index as indicator of package temp data */ + tdata->index = -1; + } else { + tdata->index = ida_alloc_max(&pdata->ida, pdata->nr_cores - 1, GFP_KERNEL); + if (tdata->index < 0) { + kfree(tdata); + return NULL; + } + pdata->core_data[tdata->index] = tdata; + } + tdata->status_reg = pkg_flag ? MSR_IA32_PACKAGE_THERM_STATUS : MSR_IA32_THERM_STATUS; - tdata->is_pkg_data = pkg_flag; tdata->cpu = cpu; tdata->cpu_core_id = topology_core_id(cpu); tdata->attr_size = MAX_CORE_ATTRS; @@ -495,6 +521,36 @@ static struct temp_data *init_temp_data(unsigned int cpu, int pkg_flag) return tdata; } +static void destroy_temp_data(struct platform_data *pdata, struct temp_data *tdata) +{ + if (is_pkg_temp_data(tdata)) { + pdata->pkg_data = NULL; + kfree(pdata->core_data); + pdata->core_data = NULL; + pdata->nr_cores = 0; + } else { + pdata->core_data[tdata->index] = NULL; + ida_free(&pdata->ida, tdata->index); + } + kfree(tdata); +} + +static struct temp_data *get_temp_data(struct platform_data *pdata, int cpu) +{ + int i; + + /* cpu < 0 means get pkg temp_data */ + if (cpu < 0) + return pdata->pkg_data; + + for (i = 0; i < pdata->nr_cores; i++) { + if (pdata->core_data[i] && + pdata->core_data[i]->cpu_core_id == topology_core_id(cpu)) + return pdata->core_data[i]; + } + return NULL; +} + static int create_core_data(struct platform_device *pdev, unsigned int cpu, int pkg_flag) { @@ -502,37 +558,19 @@ static int create_core_data(struct platform_device *pdev, unsigned int cpu, struct platform_data *pdata = platform_get_drvdata(pdev); struct cpuinfo_x86 *c = &cpu_data(cpu); u32 eax, edx; - int err, index; + int err; if (!housekeeping_cpu(cpu, HK_TYPE_MISC)) return 0; - /* - * Get the index of tdata in pdata->core_data[] - * tdata for package: pdata->core_data[1] - * tdata for core: pdata->core_data[2] .. pdata->core_data[NUM_REAL_CORES + 1] - */ - if (pkg_flag) { - index = PKG_SYSFS_ATTR_NO; - } else { - index = ida_alloc_max(&pdata->ida, NUM_REAL_CORES - 1, GFP_KERNEL); - if (index < 0) - return index; - - pdata->cpu_map[index] = topology_core_id(cpu); - index += BASE_SYSFS_ATTR_NO; - } - - tdata = init_temp_data(cpu, pkg_flag); - if (!tdata) { - err = -ENOMEM; - goto ida_free; - } + tdata = init_temp_data(pdata, cpu, pkg_flag); + if (!tdata) + return -ENOMEM; /* Test if we can access the status register */ err = rdmsr_safe_on_cpu(cpu, tdata->status_reg, &eax, &edx); if (err) - goto exit_free; + goto err; /* Make sure tdata->tjmax is a valid indicator for dynamic/static tjmax */ get_tjmax(tdata, &pdev->dev); @@ -546,20 +584,15 @@ static int create_core_data(struct platform_device *pdev, unsigned int cpu, if (get_ttarget(tdata, &pdev->dev) >= 0) tdata->attr_size++; - pdata->core_data[index] = tdata; - /* Create sysfs interfaces */ - err = create_core_attrs(tdata, pdata->hwmon_dev, index); + err = create_core_attrs(tdata, pdata->hwmon_dev); if (err) - goto exit_free; + goto err; return 0; -exit_free: - pdata->core_data[index] = NULL; - kfree(tdata); -ida_free: - if (!pkg_flag) - ida_free(&pdata->ida, index - BASE_SYSFS_ATTR_NO); + +err: + destroy_temp_data(pdata, tdata); return err; } @@ -570,10 +603,8 @@ coretemp_add_core(struct platform_device *pdev, unsigned int cpu, int pkg_flag) dev_err(&pdev->dev, "Adding Core %u failed\n", cpu); } -static void coretemp_remove_core(struct platform_data *pdata, int indx) +static void coretemp_remove_core(struct platform_data *pdata, struct temp_data *tdata) { - struct temp_data *tdata = pdata->core_data[indx]; - /* if we errored on add then this is already gone */ if (!tdata) return; @@ -581,11 +612,7 @@ static void coretemp_remove_core(struct platform_data *pdata, int indx) /* Remove the sysfs attributes */ sysfs_remove_group(&pdata->hwmon_dev->kobj, &tdata->attr_group); - kfree(pdata->core_data[indx]); - pdata->core_data[indx] = NULL; - - if (indx >= BASE_SYSFS_ATTR_NO) - ida_free(&pdata->ida, indx - BASE_SYSFS_ATTR_NO); + destroy_temp_data(pdata, tdata); } static int coretemp_device_add(int zoneid) @@ -698,7 +725,7 @@ static int coretemp_cpu_offline(unsigned int cpu) struct platform_device *pdev = coretemp_get_pdev(cpu); struct platform_data *pd; struct temp_data *tdata; - int i, indx = -1, target; + int target; /* No need to tear down any interfaces for suspend */ if (cpuhp_tasks_frozen) @@ -709,18 +736,7 @@ static int coretemp_cpu_offline(unsigned int cpu) if (!pd->hwmon_dev) return 0; - for (i = 0; i < NUM_REAL_CORES; i++) { - if (pd->cpu_map[i] == topology_core_id(cpu)) { - indx = i + BASE_SYSFS_ATTR_NO; - break; - } - } - - /* Too many cores and this core is not populated, just return */ - if (indx < 0) - return 0; - - tdata = pd->core_data[indx]; + tdata = get_temp_data(pd, cpu); cpumask_clear_cpu(cpu, &pd->cpumask); @@ -731,7 +747,7 @@ static int coretemp_cpu_offline(unsigned int cpu) */ target = cpumask_any_and(&pd->cpumask, topology_sibling_cpumask(cpu)); if (target >= nr_cpu_ids) { - coretemp_remove_core(pd, indx); + coretemp_remove_core(pd, tdata); } else if (tdata && tdata->cpu == cpu) { mutex_lock(&tdata->update_lock); tdata->cpu = target; @@ -741,10 +757,10 @@ static int coretemp_cpu_offline(unsigned int cpu) /* * If all cores in this pkg are offline, remove the interface. */ - tdata = pd->core_data[PKG_SYSFS_ATTR_NO]; + tdata = get_temp_data(pd, -1); if (cpumask_empty(&pd->cpumask)) { if (tdata) - coretemp_remove_core(pd, PKG_SYSFS_ATTR_NO); + coretemp_remove_core(pd, tdata); hwmon_device_unregister(pd->hwmon_dev); pd->hwmon_dev = NULL; return 0; diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c index 6d8c0f328b7b..4fa837e65a61 100644 --- a/drivers/hwmon/dell-smm-hwmon.c +++ b/drivers/hwmon/dell-smm-hwmon.c @@ -1450,10 +1450,15 @@ struct i8k_fan_control_data { }; enum i8k_fan_controls { + I8K_FAN_30A3_31A3, I8K_FAN_34A3_35A3, }; static const struct i8k_fan_control_data i8k_fan_control_data[] __initconst = { + [I8K_FAN_30A3_31A3] = { + .manual_fan = 0x30a3, + .auto_fan = 0x31a3, + }, [I8K_FAN_34A3_35A3] = { .manual_fan = 0x34a3, .auto_fan = 0x35a3, @@ -1517,6 +1522,14 @@ static const struct dmi_system_id i8k_whitelist_fan_control[] __initconst = { }, .driver_data = (void *)&i8k_fan_control_data[I8K_FAN_34A3_35A3], }, + { + .ident = "Dell XPS 9315", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "XPS 9315"), + }, + .driver_data = (void *)&i8k_fan_control_data[I8K_FAN_30A3_31A3], + }, { } }; diff --git a/drivers/hwmon/ds1621.c b/drivers/hwmon/ds1621.c index 21b635046521..bffbc8040171 100644 --- a/drivers/hwmon/ds1621.c +++ b/drivers/hwmon/ds1621.c @@ -380,7 +380,6 @@ MODULE_DEVICE_TABLE(i2c, ds1621_id); /* This is the driver that will be inserted */ static struct i2c_driver ds1621_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "ds1621", }, diff --git a/drivers/hwmon/ds620.c b/drivers/hwmon/ds620.c index 2b09536630cb..4fc4df012fac 100644 --- a/drivers/hwmon/ds620.c +++ b/drivers/hwmon/ds620.c @@ -241,7 +241,6 @@ MODULE_DEVICE_TABLE(i2c, ds620_id); /* This is the driver that will be inserted */ static struct i2c_driver ds620_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "ds620", }, diff --git a/drivers/hwmon/emc1403.c b/drivers/hwmon/emc1403.c index 1332e4ac078c..d370efd6f986 100644 --- a/drivers/hwmon/emc1403.c +++ b/drivers/hwmon/emc1403.c @@ -385,7 +385,7 @@ static bool emc1403_regmap_is_volatile(struct device *dev, unsigned int reg) static const struct regmap_config emc1403_regmap_config = { .reg_bits = 8, .val_bits = 8, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, .volatile_reg = emc1403_regmap_is_volatile, }; diff --git a/drivers/hwmon/emc2305.c b/drivers/hwmon/emc2305.c index 29f0e4945f19..6ef733c0be16 100644 --- a/drivers/hwmon/emc2305.c +++ b/drivers/hwmon/emc2305.c @@ -12,9 +12,6 @@ #include #include -static const unsigned short -emc2305_normal_i2c[] = { 0x27, 0x2c, 0x2d, 0x2e, 0x2f, 0x4c, 0x4d, I2C_CLIENT_END }; - #define EMC2305_REG_DRIVE_FAIL_STATUS 0x27 #define EMC2305_REG_VENDOR 0xfe #define EMC2305_FAN_MAX 0xff @@ -611,14 +608,12 @@ static void emc2305_remove(struct i2c_client *client) } static struct i2c_driver emc2305_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "emc2305", }, .probe = emc2305_probe, .remove = emc2305_remove, .id_table = emc2305_ids, - .address_list = emc2305_normal_i2c, }; module_i2c_driver(emc2305_driver); diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c index c7dd3f5b2bd5..3b259c425ab7 100644 --- a/drivers/hwmon/hwmon.c +++ b/drivers/hwmon/hwmon.c @@ -510,6 +510,7 @@ static const char * const hwmon_in_attr_templates[] = { [hwmon_in_rated_min] = "in%d_rated_min", [hwmon_in_rated_max] = "in%d_rated_max", [hwmon_in_beep] = "in%d_beep", + [hwmon_in_fault] = "in%d_fault", }; static const char * const hwmon_curr_attr_templates[] = { @@ -586,6 +587,8 @@ static const char * const hwmon_humidity_attr_templates[] = { [hwmon_humidity_fault] = "humidity%d_fault", [hwmon_humidity_rated_min] = "humidity%d_rated_min", [hwmon_humidity_rated_max] = "humidity%d_rated_max", + [hwmon_humidity_min_alarm] = "humidity%d_min_alarm", + [hwmon_humidity_max_alarm] = "humidity%d_max_alarm", }; static const char * const hwmon_fan_attr_templates[] = { diff --git a/drivers/hwmon/ina209.c b/drivers/hwmon/ina209.c index c558143e5285..d9b57a4b3e41 100644 --- a/drivers/hwmon/ina209.c +++ b/drivers/hwmon/ina209.c @@ -589,7 +589,6 @@ MODULE_DEVICE_TABLE(of, ina209_of_match); /* This is the driver that will be inserted */ static struct i2c_driver ina209_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "ina209", .of_match_table = of_match_ptr(ina209_of_match), diff --git a/drivers/hwmon/ina238.c b/drivers/hwmon/ina238.c index ca9f5d2c811b..69289293bc38 100644 --- a/drivers/hwmon/ina238.c +++ b/drivers/hwmon/ina238.c @@ -629,7 +629,6 @@ static const struct of_device_id __maybe_unused ina238_of_match[] = { MODULE_DEVICE_TABLE(of, ina238_of_match); static struct i2c_driver ina238_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "ina238", .of_match_table = of_match_ptr(ina238_of_match), diff --git a/drivers/hwmon/ina3221.c b/drivers/hwmon/ina3221.c index 5ffdc94db436..2c9530b6f192 100644 --- a/drivers/hwmon/ina3221.c +++ b/drivers/hwmon/ina3221.c @@ -762,7 +762,7 @@ static const struct regmap_config ina3221_regmap_config = { .reg_bits = 8, .val_bits = 16, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, .volatile_table = &ina3221_volatile_table, }; diff --git a/drivers/hwmon/jc42.c b/drivers/hwmon/jc42.c index f958e830b23c..75dc25df0f8b 100644 --- a/drivers/hwmon/jc42.c +++ b/drivers/hwmon/jc42.c @@ -497,7 +497,7 @@ static const struct regmap_config jc42_regmap_config = { .writeable_reg = jc42_writable_reg, .readable_reg = jc42_readable_reg, .volatile_reg = jc42_volatile_reg, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, }; static int jc42_probe(struct i2c_client *client) diff --git a/drivers/hwmon/lm83.c b/drivers/hwmon/lm83.c index 5befedca6abb..b333c9bde4e6 100644 --- a/drivers/hwmon/lm83.c +++ b/drivers/hwmon/lm83.c @@ -165,7 +165,7 @@ static bool lm83_regmap_is_volatile(struct device *dev, unsigned int reg) static const struct regmap_config lm83_regmap_config = { .reg_bits = 8, .val_bits = 8, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, .volatile_reg = lm83_regmap_is_volatile, .reg_read = lm83_regmap_reg_read, .reg_write = lm83_regmap_reg_write, diff --git a/drivers/hwmon/ltc4282.c b/drivers/hwmon/ltc4282.c new file mode 100644 index 000000000000..4f608a3790fb --- /dev/null +++ b/drivers/hwmon/ltc4282.c @@ -0,0 +1,1782 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Analog Devices LTC4282 I2C High Current Hot Swap Controller over I2C + * + * Copyright 2023 Analog Devices Inc. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LTC4282_CTRL_LSB 0x00 + #define LTC4282_CTRL_OV_RETRY_MASK BIT(0) + #define LTC4282_CTRL_UV_RETRY_MASK BIT(1) + #define LTC4282_CTRL_OC_RETRY_MASK BIT(2) + #define LTC4282_CTRL_ON_ACTIVE_LOW_MASK BIT(5) + #define LTC4282_CTRL_ON_DELAY_MASK BIT(6) +#define LTC4282_CTRL_MSB 0x01 + #define LTC4282_CTRL_VIN_MODE_MASK GENMASK(1, 0) + #define LTC4282_CTRL_OV_MODE_MASK GENMASK(3, 2) + #define LTC4282_CTRL_UV_MODE_MASK GENMASK(5, 4) +#define LTC4282_FAULT_LOG 0x04 + #define LTC4282_OV_FAULT_MASK BIT(0) + #define LTC4282_UV_FAULT_MASK BIT(1) + #define LTC4282_VDD_FAULT_MASK \ + (LTC4282_OV_FAULT_MASK | LTC4282_UV_FAULT_MASK) + #define LTC4282_OC_FAULT_MASK BIT(2) + #define LTC4282_POWER_BAD_FAULT_MASK BIT(3) + #define LTC4282_FET_SHORT_FAULT_MASK BIT(5) + #define LTC4282_FET_BAD_FAULT_MASK BIT(6) + #define LTC4282_FET_FAILURE_FAULT_MASK \ + (LTC4282_FET_SHORT_FAULT_MASK | LTC4282_FET_BAD_FAULT_MASK) +#define LTC4282_ADC_ALERT_LOG 0x05 + #define LTC4282_GPIO_ALARM_L_MASK BIT(0) + #define LTC4282_GPIO_ALARM_H_MASK BIT(1) + #define LTC4282_VSOURCE_ALARM_L_MASK BIT(2) + #define LTC4282_VSOURCE_ALARM_H_MASK BIT(3) + #define LTC4282_VSENSE_ALARM_L_MASK BIT(4) + #define LTC4282_VSENSE_ALARM_H_MASK BIT(5) + #define LTC4282_POWER_ALARM_L_MASK BIT(6) + #define LTC4282_POWER_ALARM_H_MASK BIT(7) +#define LTC4282_FET_BAD_FAULT_TIMEOUT 0x06 + #define LTC4282_FET_BAD_MAX_TIMEOUT 255 +#define LTC4282_GPIO_CONFIG 0x07 + #define LTC4282_GPIO_2_FET_STRESS_MASK BIT(1) + #define LTC4282_GPIO_1_CONFIG_MASK GENMASK(5, 4) +#define LTC4282_VGPIO_MIN 0x08 +#define LTC4282_VGPIO_MAX 0x09 +#define LTC4282_VSOURCE_MIN 0x0a +#define LTC4282_VSOURCE_MAX 0x0b +#define LTC4282_VSENSE_MIN 0x0c +#define LTC4282_VSENSE_MAX 0x0d +#define LTC4282_POWER_MIN 0x0e +#define LTC4282_POWER_MAX 0x0f +#define LTC4282_CLK_DIV 0x10 + #define LTC4282_CLK_DIV_MASK GENMASK(4, 0) + #define LTC4282_CLKOUT_MASK GENMASK(6, 5) +#define LTC4282_ILIM_ADJUST 0x11 + #define LTC4282_GPIO_MODE_MASK BIT(1) + #define LTC4282_VDD_MONITOR_MASK BIT(2) + #define LTC4282_FOLDBACK_MODE_MASK GENMASK(4, 3) + #define LTC4282_ILIM_ADJUST_MASK GENMASK(7, 5) +#define LTC4282_ENERGY 0x12 +#define LTC4282_TIME_COUNTER 0x18 +#define LTC4282_ALERT_CTRL 0x1c + #define LTC4282_ALERT_OUT_MASK BIT(6) +#define LTC4282_ADC_CTRL 0x1d + #define LTC4282_FAULT_LOG_EN_MASK BIT(2) + #define LTC4282_METER_HALT_MASK BIT(5) + #define LTC4282_METER_RESET_MASK BIT(6) + #define LTC4282_RESET_MASK BIT(7) +#define LTC4282_STATUS_LSB 0x1e + #define LTC4282_OV_STATUS_MASK BIT(0) + #define LTC4282_UV_STATUS_MASK BIT(1) + #define LTC4282_VDD_STATUS_MASK \ + (LTC4282_OV_STATUS_MASK | LTC4282_UV_STATUS_MASK) + #define LTC4282_OC_STATUS_MASK BIT(2) + #define LTC4282_POWER_GOOD_MASK BIT(3) + #define LTC4282_FET_FAILURE_MASK GENMASK(6, 5) +#define LTC4282_STATUS_MSB 0x1f +#define LTC4282_RESERVED_1 0x32 +#define LTC4282_RESERVED_2 0x33 +#define LTC4282_VGPIO 0x34 +#define LTC4282_VGPIO_LOWEST 0x36 +#define LTC4282_VGPIO_HIGHEST 0x38 +#define LTC4282_VSOURCE 0x3a +#define LTC4282_VSOURCE_LOWEST 0x3c +#define LTC4282_VSOURCE_HIGHEST 0x3e +#define LTC4282_VSENSE 0x40 +#define LTC4282_VSENSE_LOWEST 0x42 +#define LTC4282_VSENSE_HIGHEST 0x44 +#define LTC4282_POWER 0x46 +#define LTC4282_POWER_LOWEST 0x48 +#define LTC4282_POWER_HIGHEST 0x4a +#define LTC4282_RESERVED_3 0x50 + +#define LTC4282_CLKIN_MIN (250 * KILO) +#define LTC4282_CLKIN_MAX (15500 * KILO) +#define LTC4282_CLKIN_RANGE (LTC4282_CLKIN_MAX - LTC4282_CLKIN_MIN + 1) +#define LTC4282_CLKOUT_SYSTEM (250 * KILO) +#define LTC4282_CLKOUT_CNV 15 + +enum { + LTC4282_CHAN_VSOURCE, + LTC4282_CHAN_VDD, + LTC4282_CHAN_VGPIO, +}; + +struct ltc4282_cache { + u32 in_max_raw; + u32 in_min_raw; + long in_highest; + long in_lowest; + bool en; +}; + +struct ltc4282_state { + struct regmap *map; + /* Protect against multiple accesses to the device registers */ + struct mutex lock; + struct clk_hw clk_hw; + /* + * Used to cache values for VDD/VSOURCE depending which will be used + * when hwmon is not enabled for that channel. Needed because they share + * the same registers. + */ + struct ltc4282_cache in0_1_cache[LTC4282_CHAN_VGPIO]; + u32 vsense_max; + long power_max; + u32 rsense; + u16 vdd; + u16 vfs_out; + bool energy_en; +}; + +enum { + LTC4282_CLKOUT_NONE, + LTC4282_CLKOUT_INT, + LTC4282_CLKOUT_TICK, +}; + +static int ltc4282_set_rate(struct clk_hw *hw, + unsigned long rate, unsigned long parent_rate) +{ + struct ltc4282_state *st = container_of(hw, struct ltc4282_state, + clk_hw); + u32 val = LTC4282_CLKOUT_INT; + + if (rate == LTC4282_CLKOUT_CNV) + val = LTC4282_CLKOUT_TICK; + + return regmap_update_bits(st->map, LTC4282_CLK_DIV, LTC4282_CLKOUT_MASK, + FIELD_PREP(LTC4282_CLKOUT_MASK, val)); +} + +/* + * Note the 15HZ conversion rate assumes 12bit ADC which is what we are + * supporting for now. + */ +static const unsigned int ltc4282_out_rates[] = { + LTC4282_CLKOUT_CNV, LTC4282_CLKOUT_SYSTEM +}; + +static long ltc4282_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + int idx = find_closest(rate, ltc4282_out_rates, + ARRAY_SIZE(ltc4282_out_rates)); + + return ltc4282_out_rates[idx]; +} + +static unsigned long ltc4282_recalc_rate(struct clk_hw *hw, + unsigned long parent) +{ + struct ltc4282_state *st = container_of(hw, struct ltc4282_state, + clk_hw); + u32 clkdiv; + int ret; + + ret = regmap_read(st->map, LTC4282_CLK_DIV, &clkdiv); + if (ret) + return 0; + + clkdiv = FIELD_GET(LTC4282_CLKOUT_MASK, clkdiv); + if (!clkdiv) + return 0; + if (clkdiv == LTC4282_CLKOUT_INT) + return LTC4282_CLKOUT_SYSTEM; + + return LTC4282_CLKOUT_CNV; +} + +static void ltc4282_disable(struct clk_hw *clk_hw) +{ + struct ltc4282_state *st = container_of(clk_hw, struct ltc4282_state, + clk_hw); + + regmap_clear_bits(st->map, LTC4282_CLK_DIV, LTC4282_CLKOUT_MASK); +} + +static int ltc4282_read_voltage_word(const struct ltc4282_state *st, u32 reg, + u32 fs, long *val) +{ + __be16 in; + int ret; + + ret = regmap_bulk_read(st->map, reg, &in, sizeof(in)); + if (ret) + return ret; + + /* + * This is also used to calculate current in which case fs comes in + * 10 * uV. Hence the ULL usage. + */ + *val = DIV_ROUND_CLOSEST_ULL(be16_to_cpu(in) * (u64)fs, U16_MAX); + return 0; +} + +static int ltc4282_read_voltage_byte_cached(const struct ltc4282_state *st, + u32 reg, u32 fs, long *val, + u32 *cached_raw) +{ + int ret; + u32 in; + + if (cached_raw) { + in = *cached_raw; + } else { + ret = regmap_read(st->map, reg, &in); + if (ret) + return ret; + } + + *val = DIV_ROUND_CLOSEST(in * fs, U8_MAX); + return 0; +} + +static int ltc4282_read_voltage_byte(const struct ltc4282_state *st, u32 reg, + u32 fs, long *val) +{ + return ltc4282_read_voltage_byte_cached(st, reg, fs, val, NULL); +} + +static int __ltc4282_read_alarm(struct ltc4282_state *st, u32 reg, u32 mask, + long *val) +{ + u32 alarm; + int ret; + + ret = regmap_read(st->map, reg, &alarm); + if (ret) + return ret; + + *val = !!(alarm & mask); + + /* if not status/fault logs, clear the alarm after reading it */ + if (reg != LTC4282_STATUS_LSB && reg != LTC4282_FAULT_LOG) + return regmap_clear_bits(st->map, reg, mask); + + return 0; +} + +static int ltc4282_read_alarm(struct ltc4282_state *st, u32 reg, u32 mask, + long *val) +{ + guard(mutex)(&st->lock); + return __ltc4282_read_alarm(st, reg, mask, val); +} + +static int ltc4282_vdd_source_read_in(struct ltc4282_state *st, u32 channel, + long *val) +{ + guard(mutex)(&st->lock); + if (!st->in0_1_cache[channel].en) + return -ENODATA; + + return ltc4282_read_voltage_word(st, LTC4282_VSOURCE, st->vfs_out, val); +} + +static int ltc4282_vdd_source_read_hist(struct ltc4282_state *st, u32 reg, + u32 channel, long *cached, long *val) +{ + int ret; + + guard(mutex)(&st->lock); + if (!st->in0_1_cache[channel].en) { + *val = *cached; + return 0; + } + + ret = ltc4282_read_voltage_word(st, reg, st->vfs_out, val); + if (ret) + return ret; + + *cached = *val; + return 0; +} + +static int ltc4282_vdd_source_read_lim(struct ltc4282_state *st, u32 reg, + u32 channel, u32 *cached, long *val) +{ + guard(mutex)(&st->lock); + if (!st->in0_1_cache[channel].en) + return ltc4282_read_voltage_byte_cached(st, reg, st->vfs_out, + val, cached); + + return ltc4282_read_voltage_byte(st, reg, st->vfs_out, val); +} + +static int ltc4282_vdd_source_read_alm(struct ltc4282_state *st, u32 mask, + u32 channel, long *val) +{ + guard(mutex)(&st->lock); + if (!st->in0_1_cache[channel].en) { + /* + * Do this otherwise alarms can get confused because we clear + * them after reading them. So, if someone mistakenly reads + * VSOURCE right before VDD (or the other way around), we might + * get no alarm just because it was cleared when reading VSOURCE + * and had no time for a new conversion and thus having the + * alarm again. + */ + *val = 0; + return 0; + } + + return __ltc4282_read_alarm(st, LTC4282_ADC_ALERT_LOG, mask, val); +} + +static int ltc4282_read_in(struct ltc4282_state *st, u32 attr, long *val, + u32 channel) +{ + switch (attr) { + case hwmon_in_input: + if (channel == LTC4282_CHAN_VGPIO) + return ltc4282_read_voltage_word(st, LTC4282_VGPIO, + 1280, val); + + return ltc4282_vdd_source_read_in(st, channel, val); + case hwmon_in_highest: + if (channel == LTC4282_CHAN_VGPIO) + return ltc4282_read_voltage_word(st, + LTC4282_VGPIO_HIGHEST, + 1280, val); + + return ltc4282_vdd_source_read_hist(st, LTC4282_VSOURCE_HIGHEST, + channel, + &st->in0_1_cache[channel].in_highest, val); + case hwmon_in_lowest: + if (channel == LTC4282_CHAN_VGPIO) + return ltc4282_read_voltage_word(st, LTC4282_VGPIO_LOWEST, + 1280, val); + + return ltc4282_vdd_source_read_hist(st, LTC4282_VSOURCE_LOWEST, + channel, + &st->in0_1_cache[channel].in_lowest, val); + case hwmon_in_max_alarm: + if (channel == LTC4282_CHAN_VGPIO) + return ltc4282_read_alarm(st, LTC4282_ADC_ALERT_LOG, + LTC4282_GPIO_ALARM_H_MASK, + val); + + return ltc4282_vdd_source_read_alm(st, + LTC4282_VSOURCE_ALARM_H_MASK, + channel, val); + case hwmon_in_min_alarm: + if (channel == LTC4282_CHAN_VGPIO) + ltc4282_read_alarm(st, LTC4282_ADC_ALERT_LOG, + LTC4282_GPIO_ALARM_L_MASK, val); + + return ltc4282_vdd_source_read_alm(st, + LTC4282_VSOURCE_ALARM_L_MASK, + channel, val); + case hwmon_in_crit_alarm: + return ltc4282_read_alarm(st, LTC4282_STATUS_LSB, + LTC4282_OV_STATUS_MASK, val); + case hwmon_in_lcrit_alarm: + return ltc4282_read_alarm(st, LTC4282_STATUS_LSB, + LTC4282_UV_STATUS_MASK, val); + case hwmon_in_max: + if (channel == LTC4282_CHAN_VGPIO) + return ltc4282_read_voltage_byte(st, LTC4282_VGPIO_MAX, + 1280, val); + + return ltc4282_vdd_source_read_lim(st, LTC4282_VSOURCE_MAX, + channel, + &st->in0_1_cache[channel].in_max_raw, val); + case hwmon_in_min: + if (channel == LTC4282_CHAN_VGPIO) + return ltc4282_read_voltage_byte(st, LTC4282_VGPIO_MIN, + 1280, val); + + return ltc4282_vdd_source_read_lim(st, LTC4282_VSOURCE_MIN, + channel, + &st->in0_1_cache[channel].in_min_raw, val); + case hwmon_in_enable: + scoped_guard(mutex, &st->lock) { + *val = st->in0_1_cache[channel].en; + } + return 0; + case hwmon_in_fault: + /* + * We report failure if we detect either a fer_bad or a + * fet_short in the status register. + */ + return ltc4282_read_alarm(st, LTC4282_STATUS_LSB, + LTC4282_FET_FAILURE_MASK, val); + default: + return -EOPNOTSUPP; + } +} + +static int ltc4282_read_current_word(const struct ltc4282_state *st, u32 reg, + long *val) +{ + long in; + int ret; + + /* + * We pass in full scale in 10 * micro (note that 40 is already + * millivolt) so we have better approximations to calculate current. + */ + ret = ltc4282_read_voltage_word(st, reg, DECA * 40 * MILLI, &in); + if (ret) + return ret; + + *val = DIV_ROUND_CLOSEST(in * MILLI, st->rsense); + + return 0; +} + +static int ltc4282_read_current_byte(const struct ltc4282_state *st, u32 reg, + long *val) +{ + long in; + int ret; + + ret = ltc4282_read_voltage_byte(st, reg, DECA * 40 * MILLI, &in); + if (ret) + return ret; + + *val = DIV_ROUND_CLOSEST(in * MILLI, st->rsense); + + return 0; +} + +static int ltc4282_read_curr(struct ltc4282_state *st, const u32 attr, + long *val) +{ + switch (attr) { + case hwmon_curr_input: + return ltc4282_read_current_word(st, LTC4282_VSENSE, val); + case hwmon_curr_highest: + return ltc4282_read_current_word(st, LTC4282_VSENSE_HIGHEST, + val); + case hwmon_curr_lowest: + return ltc4282_read_current_word(st, LTC4282_VSENSE_LOWEST, + val); + case hwmon_curr_max: + return ltc4282_read_current_byte(st, LTC4282_VSENSE_MAX, val); + case hwmon_curr_min: + return ltc4282_read_current_byte(st, LTC4282_VSENSE_MIN, val); + case hwmon_curr_max_alarm: + return ltc4282_read_alarm(st, LTC4282_ADC_ALERT_LOG, + LTC4282_VSENSE_ALARM_H_MASK, val); + case hwmon_curr_min_alarm: + return ltc4282_read_alarm(st, LTC4282_ADC_ALERT_LOG, + LTC4282_VSENSE_ALARM_L_MASK, val); + case hwmon_curr_crit_alarm: + return ltc4282_read_alarm(st, LTC4282_STATUS_LSB, + LTC4282_OC_STATUS_MASK, val); + default: + return -EOPNOTSUPP; + } +} + +static int ltc4282_read_power_word(const struct ltc4282_state *st, u32 reg, + long *val) +{ + u64 temp = DECA * 40ULL * st->vfs_out * BIT(16), temp_2; + __be16 raw; + u16 power; + int ret; + + ret = regmap_bulk_read(st->map, reg, &raw, sizeof(raw)); + if (ret) + return ret; + + power = be16_to_cpu(raw); + /* + * Power is given by: + * P = CODE(16b) * 0.040 * Vfs(out) * 2^16 / ((2^16 - 1)^2 * Rsense) + */ + if (check_mul_overflow(power * temp, MICRO, &temp_2)) { + temp = DIV_ROUND_CLOSEST_ULL(power * temp, U16_MAX); + *val = DIV64_U64_ROUND_CLOSEST(temp * MICRO, + U16_MAX * (u64)st->rsense); + return 0; + } + + *val = DIV64_U64_ROUND_CLOSEST(temp_2, + st->rsense * int_pow(U16_MAX, 2)); + + return 0; +} + +static int ltc4282_read_power_byte(const struct ltc4282_state *st, u32 reg, + long *val) +{ + u32 power; + u64 temp; + int ret; + + ret = regmap_read(st->map, reg, &power); + if (ret) + return ret; + + temp = power * 40 * DECA * st->vfs_out * BIT_ULL(8); + *val = DIV64_U64_ROUND_CLOSEST(temp * MICRO, + int_pow(U8_MAX, 2) * st->rsense); + + return 0; +} + +static int ltc4282_read_energy(const struct ltc4282_state *st, u64 *val) +{ + u64 temp, energy; + __be64 raw; + int ret; + + ret = regmap_bulk_read(st->map, LTC4282_ENERGY, &raw, 6); + if (ret) + return ret; + + energy = be64_to_cpu(raw) >> 16; + /* + * The formula for energy is given by: + * E = CODE(48b) * 0.040 * Vfs(out) * Tconv * 256 / + * ((2^16 - 1)^2 * Rsense) + * + * Since we only support 12bit ADC, Tconv = 0.065535s. Passing Vfs(out) + * and 0.040 to mV and Tconv to us, we can simplify the formula to: + * E = CODE(48b) * 40 * Vfs(out) * 256 / (U16_MAX * Rsense) + * + * As Rsense can have tenths of micro-ohm resolution, we need to + * multiply by DECA to get microujoule. + */ + if (check_mul_overflow(DECA * st->vfs_out * 40 * BIT(8), energy, &temp)) { + temp = DIV_ROUND_CLOSEST(DECA * st->vfs_out * 40 * BIT(8), U16_MAX); + *val = DIV_ROUND_CLOSEST_ULL(temp * energy, st->rsense); + return 0; + } + + *val = DIV64_U64_ROUND_CLOSEST(temp, U16_MAX * (u64)st->rsense); + + return 0; +} + +static int ltc4282_read_power(struct ltc4282_state *st, const u32 attr, + long *val) +{ + switch (attr) { + case hwmon_power_input: + return ltc4282_read_power_word(st, LTC4282_POWER, val); + case hwmon_power_input_highest: + return ltc4282_read_power_word(st, LTC4282_POWER_HIGHEST, val); + case hwmon_power_input_lowest: + return ltc4282_read_power_word(st, LTC4282_POWER_LOWEST, val); + case hwmon_power_max_alarm: + return ltc4282_read_alarm(st, LTC4282_ADC_ALERT_LOG, + LTC4282_POWER_ALARM_H_MASK, val); + case hwmon_power_min_alarm: + return ltc4282_read_alarm(st, LTC4282_ADC_ALERT_LOG, + LTC4282_POWER_ALARM_L_MASK, val); + case hwmon_power_max: + return ltc4282_read_power_byte(st, LTC4282_POWER_MAX, val); + case hwmon_power_min: + return ltc4282_read_power_byte(st, LTC4282_POWER_MIN, val); + default: + return -EOPNOTSUPP; + } +} + +static int ltc4282_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *val) +{ + struct ltc4282_state *st = dev_get_drvdata(dev); + + switch (type) { + case hwmon_in: + return ltc4282_read_in(st, attr, val, channel); + case hwmon_curr: + return ltc4282_read_curr(st, attr, val); + case hwmon_power: + return ltc4282_read_power(st, attr, val); + case hwmon_energy: + scoped_guard(mutex, &st->lock) { + *val = st->energy_en; + } + return 0; + default: + return -EOPNOTSUPP; + } +} + +static int ltc4282_write_power_byte(const struct ltc4282_state *st, u32 reg, + long val) +{ + u32 power; + u64 temp; + + if (val > st->power_max) + val = st->power_max; + + temp = val * int_pow(U8_MAX, 2) * st->rsense; + power = DIV64_U64_ROUND_CLOSEST(temp, + MICRO * DECA * 256ULL * st->vfs_out * 40); + + return regmap_write(st->map, reg, power); +} + +static int ltc4282_write_power_word(const struct ltc4282_state *st, u32 reg, + long val) +{ + u64 temp = int_pow(U16_MAX, 2) * st->rsense, temp_2; + __be16 __raw; + u16 code; + + if (check_mul_overflow(temp, val, &temp_2)) { + temp = DIV_ROUND_CLOSEST_ULL(temp, DECA * MICRO); + code = DIV64_U64_ROUND_CLOSEST(temp * val, + 40ULL * BIT(16) * st->vfs_out); + } else { + temp = DECA * MICRO * 40ULL * BIT(16) * st->vfs_out; + code = DIV64_U64_ROUND_CLOSEST(temp_2, temp); + } + + __raw = cpu_to_be16(code); + return regmap_bulk_write(st->map, reg, &__raw, sizeof(__raw)); +} + +static int __ltc4282_in_write_history(const struct ltc4282_state *st, u32 reg, + long lowest, long highest, u32 fs) +{ + __be16 __raw; + u16 tmp; + int ret; + + tmp = DIV_ROUND_CLOSEST(U16_MAX * lowest, fs); + + __raw = cpu_to_be16(tmp); + + ret = regmap_bulk_write(st->map, reg, &__raw, 2); + if (ret) + return ret; + + tmp = DIV_ROUND_CLOSEST(U16_MAX * highest, fs); + + __raw = cpu_to_be16(tmp); + + return regmap_bulk_write(st->map, reg + 2, &__raw, 2); +} + +static int ltc4282_in_write_history(struct ltc4282_state *st, u32 reg, + long lowest, long highest, u32 fs) +{ + guard(mutex)(&st->lock); + return __ltc4282_in_write_history(st, reg, lowest, highest, fs); +} + +static int ltc4282_power_reset_hist(struct ltc4282_state *st) +{ + int ret; + + guard(mutex)(&st->lock); + + ret = ltc4282_write_power_word(st, LTC4282_POWER_LOWEST, + st->power_max); + if (ret) + return ret; + + ret = ltc4282_write_power_word(st, LTC4282_POWER_HIGHEST, 0); + if (ret) + return ret; + + /* now, let's also clear possible power_bad fault logs */ + return regmap_clear_bits(st->map, LTC4282_FAULT_LOG, + LTC4282_POWER_BAD_FAULT_MASK); +} + +static int ltc4282_write_power(struct ltc4282_state *st, u32 attr, + long val) +{ + switch (attr) { + case hwmon_power_max: + return ltc4282_write_power_byte(st, LTC4282_POWER_MAX, val); + case hwmon_power_min: + return ltc4282_write_power_byte(st, LTC4282_POWER_MIN, val); + case hwmon_power_reset_history: + return ltc4282_power_reset_hist(st); + default: + return -EOPNOTSUPP; + } +} + +static int ltc4282_write_voltage_byte_cached(const struct ltc4282_state *st, + u32 reg, u32 fs, long val, + u32 *cache_raw) +{ + u32 in; + + val = clamp_val(val, 0, fs); + in = DIV_ROUND_CLOSEST(val * U8_MAX, fs); + + if (cache_raw) { + *cache_raw = in; + return 0; + } + + return regmap_write(st->map, reg, in); +} + +static int ltc4282_write_voltage_byte(const struct ltc4282_state *st, u32 reg, + u32 fs, long val) +{ + return ltc4282_write_voltage_byte_cached(st, reg, fs, val, NULL); +} + +static int ltc4282_cache_history(struct ltc4282_state *st, u32 channel) +{ + long val; + int ret; + + ret = ltc4282_read_voltage_word(st, LTC4282_VSOURCE_LOWEST, st->vfs_out, + &val); + if (ret) + return ret; + + st->in0_1_cache[channel].in_lowest = val; + + ret = ltc4282_read_voltage_word(st, LTC4282_VSOURCE_HIGHEST, + st->vfs_out, &val); + if (ret) + return ret; + + st->in0_1_cache[channel].in_highest = val; + + ret = regmap_read(st->map, LTC4282_VSOURCE_MIN, + &st->in0_1_cache[channel].in_min_raw); + if (ret) + return ret; + + return regmap_read(st->map, LTC4282_VSOURCE_MAX, + &st->in0_1_cache[channel].in_max_raw); +} + +static int ltc4282_cache_sync(struct ltc4282_state *st, u32 channel) +{ + int ret; + + ret = __ltc4282_in_write_history(st, LTC4282_VSOURCE_LOWEST, + st->in0_1_cache[channel].in_lowest, + st->in0_1_cache[channel].in_highest, + st->vfs_out); + if (ret) + return ret; + + ret = regmap_write(st->map, LTC4282_VSOURCE_MIN, + st->in0_1_cache[channel].in_min_raw); + if (ret) + return ret; + + return regmap_write(st->map, LTC4282_VSOURCE_MAX, + st->in0_1_cache[channel].in_max_raw); +} + +static int ltc4282_vdd_source_write_lim(struct ltc4282_state *st, u32 reg, + int channel, u32 *cache, long val) +{ + int ret; + + guard(mutex)(&st->lock); + if (st->in0_1_cache[channel].en) + ret = ltc4282_write_voltage_byte(st, reg, st->vfs_out, val); + else + ret = ltc4282_write_voltage_byte_cached(st, reg, st->vfs_out, + val, cache); + + return ret; +} + +static int ltc4282_vdd_source_reset_hist(struct ltc4282_state *st, int channel) +{ + long lowest = st->vfs_out; + int ret; + + if (channel == LTC4282_CHAN_VDD) + lowest = st->vdd; + + guard(mutex)(&st->lock); + if (st->in0_1_cache[channel].en) { + ret = __ltc4282_in_write_history(st, LTC4282_VSOURCE_LOWEST, + lowest, 0, st->vfs_out); + if (ret) + return ret; + } + + st->in0_1_cache[channel].in_lowest = lowest; + st->in0_1_cache[channel].in_highest = 0; + + /* + * We are also clearing possible fault logs in reset_history. Clearing + * the logs might be important when the auto retry bits are not enabled + * as the chip only enables the output again after having these logs + * cleared. As some of these logs are related to limits, it makes sense + * to clear them in here. For VDD, we need to clear under/over voltage + * events. For VSOURCE, fet_short and fet_bad... + */ + if (channel == LTC4282_CHAN_VSOURCE) + return regmap_clear_bits(st->map, LTC4282_FAULT_LOG, + LTC4282_FET_FAILURE_FAULT_MASK); + + return regmap_clear_bits(st->map, LTC4282_FAULT_LOG, + LTC4282_VDD_FAULT_MASK); +} + +/* + * We need to mux between VSOURCE and VDD which means they are mutually + * exclusive. Moreover, we can't really disable both VDD and VSOURCE as the ADC + * is continuously running (we cannot independently halt it without also + * stopping VGPIO). Hence, the logic is that disabling or enabling VDD will + * automatically have the reverse effect on VSOURCE and vice-versa. + */ +static int ltc4282_vdd_source_enable(struct ltc4282_state *st, int channel, + long val) +{ + int ret, other_chan = ~channel & 0x1; + u8 __val = val; + + guard(mutex)(&st->lock); + if (st->in0_1_cache[channel].en == !!val) + return 0; + + /* clearing the bit makes the ADC to monitor VDD */ + if (channel == LTC4282_CHAN_VDD) + __val = !__val; + + ret = regmap_update_bits(st->map, LTC4282_ILIM_ADJUST, + LTC4282_VDD_MONITOR_MASK, + FIELD_PREP(LTC4282_VDD_MONITOR_MASK, !!__val)); + if (ret) + return ret; + + st->in0_1_cache[channel].en = !!val; + st->in0_1_cache[other_chan].en = !val; + + if (st->in0_1_cache[channel].en) { + /* + * Then, we are disabling @other_chan. Let's save it's current + * history. + */ + ret = ltc4282_cache_history(st, other_chan); + if (ret) + return ret; + + return ltc4282_cache_sync(st, channel); + } + /* + * Then, we are enabling @other_chan. We need to do the opposite from + * above. + */ + ret = ltc4282_cache_history(st, channel); + if (ret) + return ret; + + return ltc4282_cache_sync(st, other_chan); +} + +static int ltc4282_write_in(struct ltc4282_state *st, u32 attr, long val, + int channel) +{ + switch (attr) { + case hwmon_in_max: + if (channel == LTC4282_CHAN_VGPIO) + return ltc4282_write_voltage_byte(st, LTC4282_VGPIO_MAX, + 1280, val); + + return ltc4282_vdd_source_write_lim(st, LTC4282_VSOURCE_MAX, + channel, + &st->in0_1_cache[channel].in_max_raw, val); + case hwmon_in_min: + if (channel == LTC4282_CHAN_VGPIO) + return ltc4282_write_voltage_byte(st, LTC4282_VGPIO_MIN, + 1280, val); + + return ltc4282_vdd_source_write_lim(st, LTC4282_VSOURCE_MIN, + channel, + &st->in0_1_cache[channel].in_min_raw, val); + case hwmon_in_reset_history: + if (channel == LTC4282_CHAN_VGPIO) + return ltc4282_in_write_history(st, + LTC4282_VGPIO_LOWEST, + 1280, 0, 1280); + + return ltc4282_vdd_source_reset_hist(st, channel); + case hwmon_in_enable: + return ltc4282_vdd_source_enable(st, channel, val); + default: + return -EOPNOTSUPP; + } +} + +static int ltc4282_curr_reset_hist(struct ltc4282_state *st) +{ + int ret; + + guard(mutex)(&st->lock); + + ret = __ltc4282_in_write_history(st, LTC4282_VSENSE_LOWEST, + st->vsense_max, 0, 40 * MILLI); + if (ret) + return ret; + + /* now, let's also clear possible overcurrent fault logs */ + return regmap_clear_bits(st->map, LTC4282_FAULT_LOG, + LTC4282_OC_FAULT_MASK); +} + +static int ltc4282_write_curr(struct ltc4282_state *st, u32 attr, + long val) +{ + /* need to pass it in millivolt */ + u32 in = DIV_ROUND_CLOSEST_ULL((u64)val * st->rsense, DECA * MICRO); + + switch (attr) { + case hwmon_curr_max: + return ltc4282_write_voltage_byte(st, LTC4282_VSENSE_MAX, 40, + in); + case hwmon_curr_min: + return ltc4282_write_voltage_byte(st, LTC4282_VSENSE_MIN, 40, + in); + case hwmon_curr_reset_history: + return ltc4282_curr_reset_hist(st); + default: + return -EOPNOTSUPP; + } +} + +static int ltc4282_energy_enable_set(struct ltc4282_state *st, long val) +{ + int ret; + + guard(mutex)(&st->lock); + /* setting the bit halts the meter */ + ret = regmap_update_bits(st->map, LTC4282_ADC_CTRL, + LTC4282_METER_HALT_MASK, + FIELD_PREP(LTC4282_METER_HALT_MASK, !val)); + if (ret) + return ret; + + st->energy_en = !!val; + + return 0; +} + +static int ltc4282_write(struct device *dev, + enum hwmon_sensor_types type, + u32 attr, int channel, long val) +{ + struct ltc4282_state *st = dev_get_drvdata(dev); + + switch (type) { + case hwmon_power: + return ltc4282_write_power(st, attr, val); + case hwmon_in: + return ltc4282_write_in(st, attr, val, channel); + case hwmon_curr: + return ltc4282_write_curr(st, attr, val); + case hwmon_energy: + return ltc4282_energy_enable_set(st, val); + default: + return -EOPNOTSUPP; + } +} + +static umode_t ltc4282_in_is_visible(const struct ltc4282_state *st, u32 attr) +{ + switch (attr) { + case hwmon_in_input: + case hwmon_in_highest: + case hwmon_in_lowest: + case hwmon_in_max_alarm: + case hwmon_in_min_alarm: + case hwmon_in_label: + case hwmon_in_lcrit_alarm: + case hwmon_in_crit_alarm: + case hwmon_in_fault: + return 0444; + case hwmon_in_max: + case hwmon_in_min: + case hwmon_in_enable: + case hwmon_in_reset_history: + return 0644; + default: + return 0; + } +} + +static umode_t ltc4282_curr_is_visible(u32 attr) +{ + switch (attr) { + case hwmon_curr_input: + case hwmon_curr_highest: + case hwmon_curr_lowest: + case hwmon_curr_max_alarm: + case hwmon_curr_min_alarm: + case hwmon_curr_crit_alarm: + case hwmon_curr_label: + return 0444; + case hwmon_curr_max: + case hwmon_curr_min: + case hwmon_curr_reset_history: + return 0644; + default: + return 0; + } +} + +static umode_t ltc4282_power_is_visible(u32 attr) +{ + switch (attr) { + case hwmon_power_input: + case hwmon_power_input_highest: + case hwmon_power_input_lowest: + case hwmon_power_label: + case hwmon_power_max_alarm: + case hwmon_power_min_alarm: + return 0444; + case hwmon_power_max: + case hwmon_power_min: + case hwmon_power_reset_history: + return 0644; + default: + return 0; + } +} + +static umode_t ltc4282_is_visible(const void *data, + enum hwmon_sensor_types type, + u32 attr, int channel) +{ + switch (type) { + case hwmon_in: + return ltc4282_in_is_visible(data, attr); + case hwmon_curr: + return ltc4282_curr_is_visible(attr); + case hwmon_power: + return ltc4282_power_is_visible(attr); + case hwmon_energy: + /* hwmon_energy_enable */ + return 0644; + default: + return 0; + } +} + +static const char * const ltc4282_in_strs[] = { + "VSOURCE", "VDD", "VGPIO" +}; + +static int ltc4282_read_labels(struct device *dev, + enum hwmon_sensor_types type, + u32 attr, int channel, const char **str) +{ + switch (type) { + case hwmon_in: + *str = ltc4282_in_strs[channel]; + return 0; + case hwmon_curr: + *str = "ISENSE"; + return 0; + case hwmon_power: + *str = "Power"; + return 0; + default: + return -EOPNOTSUPP; + } +} + +static ssize_t ltc4282_energy_show(struct device *dev, + struct device_attribute *da, char *buf) +{ + struct ltc4282_state *st = dev_get_drvdata(dev); + u64 energy; + int ret; + + guard(mutex)(&st->lock); + if (!st->energy_en) + return -ENODATA; + + ret = ltc4282_read_energy(st, &energy); + if (ret < 0) + return ret; + + return sysfs_emit(buf, "%llu\n", energy); +} + +static const struct clk_ops ltc4282_ops = { + .recalc_rate = ltc4282_recalc_rate, + .round_rate = ltc4282_round_rate, + .set_rate = ltc4282_set_rate, + .disable = ltc4282_disable, +}; + +static int ltc428_clk_provider_setup(struct ltc4282_state *st, + struct device *dev) +{ + struct clk_init_data init; + int ret; + + if (!IS_ENABLED(CONFIG_COMMON_CLK)) + return 0; + + init.name = devm_kasprintf(dev, GFP_KERNEL, "%s-clk", + fwnode_get_name(dev_fwnode(dev))); + if (!init.name) + return -ENOMEM; + + init.ops = <c4282_ops; + init.flags = CLK_GET_RATE_NOCACHE; + st->clk_hw.init = &init; + + ret = devm_clk_hw_register(dev, &st->clk_hw); + if (ret) + return ret; + + return devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get, + &st->clk_hw); +} + +static int ltc428_clks_setup(struct ltc4282_state *st, struct device *dev) +{ + unsigned long rate; + struct clk *clkin; + u32 val; + int ret; + + ret = ltc428_clk_provider_setup(st, dev); + if (ret) + return ret; + + clkin = devm_clk_get_optional_enabled(dev, NULL); + if (IS_ERR(clkin)) + return dev_err_probe(dev, PTR_ERR(clkin), + "Failed to get clkin"); + if (!clkin) + return 0; + + rate = clk_get_rate(clkin); + if (!in_range(rate, LTC4282_CLKIN_MIN, LTC4282_CLKIN_RANGE)) + return dev_err_probe(dev, -EINVAL, + "Invalid clkin range(%lu) [%lu %lu]\n", + rate, LTC4282_CLKIN_MIN, + LTC4282_CLKIN_MAX); + + /* + * Clocks faster than 250KHZ should be reduced to 250KHZ. The clock + * frequency is divided by twice the value in the register. + */ + val = rate / (2 * LTC4282_CLKIN_MIN); + + return regmap_update_bits(st->map, LTC4282_CLK_DIV, + LTC4282_CLK_DIV_MASK, + FIELD_PREP(LTC4282_CLK_DIV_MASK, val)); +} + +static const int ltc4282_curr_lim_uv[] = { + 12500, 15625, 18750, 21875, 25000, 28125, 31250, 34375 +}; + +static int ltc4282_get_defaults(struct ltc4282_state *st, u32 *vin_mode) +{ + u32 reg_val, ilm_adjust; + int ret; + + ret = regmap_read(st->map, LTC4282_ADC_CTRL, ®_val); + if (ret) + return ret; + + st->energy_en = !FIELD_GET(LTC4282_METER_HALT_MASK, reg_val); + + ret = regmap_read(st->map, LTC4282_CTRL_MSB, ®_val); + if (ret) + return ret; + + *vin_mode = FIELD_GET(LTC4282_CTRL_VIN_MODE_MASK, reg_val); + + ret = regmap_read(st->map, LTC4282_ILIM_ADJUST, ®_val); + if (ret) + return ret; + + ilm_adjust = FIELD_GET(LTC4282_ILIM_ADJUST_MASK, reg_val); + st->vsense_max = ltc4282_curr_lim_uv[ilm_adjust]; + + st->in0_1_cache[LTC4282_CHAN_VSOURCE].en = FIELD_GET(LTC4282_VDD_MONITOR_MASK, + ilm_adjust); + if (!st->in0_1_cache[LTC4282_CHAN_VSOURCE].en) { + st->in0_1_cache[LTC4282_CHAN_VDD].en = true; + return regmap_read(st->map, LTC4282_VSOURCE_MAX, + &st->in0_1_cache[LTC4282_CHAN_VSOURCE].in_max_raw); + } + + return regmap_read(st->map, LTC4282_VSOURCE_MAX, + &st->in0_1_cache[LTC4282_CHAN_VDD].in_max_raw); +} + +/* + * Set max limits for ISENSE and Power as that depends on the max voltage on + * rsense that is defined in ILIM_ADJUST. This is specially important for power + * because for some rsense and vfsout values, if we allow the default raw 255 + * value, that would overflow long in 32bit archs when reading back the max + * power limit. + * + * Also set meaningful historic values for VDD and VSOURCE + * (0 would not mean much). + */ +static int ltc4282_set_max_limits(struct ltc4282_state *st) +{ + int ret; + + ret = ltc4282_write_voltage_byte(st, LTC4282_VSENSE_MAX, 40 * MILLI, + st->vsense_max); + if (ret) + return ret; + + /* Power is given by ISENSE * Vout. */ + st->power_max = DIV_ROUND_CLOSEST(st->vsense_max * DECA * MILLI, st->rsense) * st->vfs_out; + ret = ltc4282_write_power_byte(st, LTC4282_POWER_MAX, st->power_max); + if (ret) + return ret; + + if (st->in0_1_cache[LTC4282_CHAN_VDD].en) { + st->in0_1_cache[LTC4282_CHAN_VSOURCE].in_lowest = st->vfs_out; + return __ltc4282_in_write_history(st, LTC4282_VSOURCE_LOWEST, + st->vdd, 0, st->vfs_out); + } + + st->in0_1_cache[LTC4282_CHAN_VDD].in_lowest = st->vdd; + return __ltc4282_in_write_history(st, LTC4282_VSOURCE_LOWEST, + st->vfs_out, 0, st->vfs_out); +} + +static const char * const ltc4282_gpio1_modes[] = { + "power_bad", "power_good" +}; + +static const char * const ltc4282_gpio2_modes[] = { + "adc_input", "stress_fet" +}; + +static int ltc4282_gpio_setup(struct ltc4282_state *st, struct device *dev) +{ + const char *func = NULL; + int ret; + + ret = device_property_read_string(dev, "adi,gpio1-mode", &func); + if (!ret) { + ret = match_string(ltc4282_gpio1_modes, + ARRAY_SIZE(ltc4282_gpio1_modes), func); + if (ret < 0) + return dev_err_probe(dev, ret, + "Invalid func(%s) for gpio1\n", + func); + + ret = regmap_update_bits(st->map, LTC4282_GPIO_CONFIG, + LTC4282_GPIO_1_CONFIG_MASK, + FIELD_PREP(LTC4282_GPIO_1_CONFIG_MASK, ret)); + if (ret) + return ret; + } + + ret = device_property_read_string(dev, "adi,gpio2-mode", &func); + if (!ret) { + ret = match_string(ltc4282_gpio2_modes, + ARRAY_SIZE(ltc4282_gpio2_modes), func); + if (ret < 0) + return dev_err_probe(dev, ret, + "Invalid func(%s) for gpio2\n", + func); + if (!ret) { + /* setting the bit to 1 so the ADC to monitors GPIO2 */ + ret = regmap_set_bits(st->map, LTC4282_ILIM_ADJUST, + LTC4282_GPIO_MODE_MASK); + } else { + ret = regmap_update_bits(st->map, LTC4282_GPIO_CONFIG, + LTC4282_GPIO_2_FET_STRESS_MASK, + FIELD_PREP(LTC4282_GPIO_2_FET_STRESS_MASK, 1)); + } + + if (ret) + return ret; + } + + if (!device_property_read_bool(dev, "adi,gpio3-monitor-enable")) + return 0; + + if (func && !strcmp(func, "adc_input")) + return dev_err_probe(dev, -EINVAL, + "Cannot have both gpio2 and gpio3 muxed into the ADC"); + + return regmap_clear_bits(st->map, LTC4282_ILIM_ADJUST, + LTC4282_GPIO_MODE_MASK); +} + +static const char * const ltc4282_dividers[] = { + "external", "vdd_5_percent", "vdd_10_percent", "vdd_15_percent" +}; + +/* This maps the Vout full scale for the given Vin mode */ +static const u16 ltc4282_vfs_milli[] = { 5540, 8320, 16640, 33280 }; + +static const u16 ltc4282_vdd_milli[] = { 3300, 5000, 12000, 24000 }; + +enum { + LTC4282_VIN_3_3V, + LTC4282_VIN_5V, + LTC4282_VIN_12V, + LTC4282_VIN_24V, +}; + +static int ltc4282_setup(struct ltc4282_state *st, struct device *dev) +{ + const char *divider; + u32 val, vin_mode; + int ret; + + /* The part has an eeprom so let's get the needed defaults from it */ + ret = ltc4282_get_defaults(st, &vin_mode); + if (ret) + return ret; + + ret = device_property_read_u32(dev, "adi,rsense-nano-ohms", + &st->rsense); + if (ret) + return dev_err_probe(dev, ret, + "Failed to read adi,rsense-nano-ohms\n"); + if (st->rsense < CENTI) + return dev_err_probe(dev, -EINVAL, + "adi,rsense-nano-ohms too small (< %lu)\n", + CENTI); + + /* + * The resolution for rsense is tenths of micro (eg: 62.5 uOhm) which + * means we need nano in the bindings. However, to make things easier to + * handle (with respect to overflows) we divide it by 100 as we don't + * really need the last two digits. + */ + st->rsense /= CENTI; + + val = vin_mode; + ret = device_property_read_u32(dev, "adi,vin-mode-microvolt", &val); + if (!ret) { + switch (val) { + case 3300000: + val = LTC4282_VIN_3_3V; + break; + case 5000000: + val = LTC4282_VIN_5V; + break; + case 12000000: + val = LTC4282_VIN_12V; + break; + case 24000000: + val = LTC4282_VIN_24V; + break; + default: + return dev_err_probe(dev, -EINVAL, + "Invalid val(%u) for vin-mode-microvolt\n", + val); + } + + ret = regmap_update_bits(st->map, LTC4282_CTRL_MSB, + LTC4282_CTRL_VIN_MODE_MASK, + FIELD_PREP(LTC4282_CTRL_VIN_MODE_MASK, val)); + if (ret) + return ret; + + /* Foldback mode should also be set to the input voltage */ + ret = regmap_update_bits(st->map, LTC4282_ILIM_ADJUST, + LTC4282_FOLDBACK_MODE_MASK, + FIELD_PREP(LTC4282_FOLDBACK_MODE_MASK, val)); + if (ret) + return ret; + } + + st->vfs_out = ltc4282_vfs_milli[val]; + st->vdd = ltc4282_vdd_milli[val]; + + ret = device_property_read_u32(dev, "adi,current-limit-sense-microvolt", + &st->vsense_max); + if (!ret) { + int reg_val; + + switch (val) { + case 12500: + reg_val = 0; + break; + case 15625: + reg_val = 1; + break; + case 18750: + reg_val = 2; + break; + case 21875: + reg_val = 3; + break; + case 25000: + reg_val = 4; + break; + case 28125: + reg_val = 5; + break; + case 31250: + reg_val = 6; + break; + case 34375: + reg_val = 7; + break; + default: + return dev_err_probe(dev, -EINVAL, + "Invalid val(%u) for adi,current-limit-microvolt\n", + st->vsense_max); + } + + ret = regmap_update_bits(st->map, LTC4282_ILIM_ADJUST, + LTC4282_ILIM_ADJUST_MASK, + FIELD_PREP(LTC4282_ILIM_ADJUST_MASK, reg_val)); + if (ret) + return ret; + } + + ret = ltc4282_set_max_limits(st); + if (ret) + return ret; + + ret = device_property_read_string(dev, "adi,overvoltage-dividers", + ÷r); + if (!ret) { + int div = match_string(ltc4282_dividers, + ARRAY_SIZE(ltc4282_dividers), divider); + if (div < 0) + return dev_err_probe(dev, -EINVAL, + "Invalid val(%s) for adi,overvoltage-divider\n", + divider); + + ret = regmap_update_bits(st->map, LTC4282_CTRL_MSB, + LTC4282_CTRL_OV_MODE_MASK, + FIELD_PREP(LTC4282_CTRL_OV_MODE_MASK, div)); + } + + ret = device_property_read_string(dev, "adi,undervoltage-dividers", + ÷r); + if (!ret) { + int div = match_string(ltc4282_dividers, + ARRAY_SIZE(ltc4282_dividers), divider); + if (div < 0) + return dev_err_probe(dev, -EINVAL, + "Invalid val(%s) for adi,undervoltage-divider\n", + divider); + + ret = regmap_update_bits(st->map, LTC4282_CTRL_MSB, + LTC4282_CTRL_UV_MODE_MASK, + FIELD_PREP(LTC4282_CTRL_UV_MODE_MASK, div)); + } + + if (device_property_read_bool(dev, "adi,overcurrent-retry")) { + ret = regmap_set_bits(st->map, LTC4282_CTRL_LSB, + LTC4282_CTRL_OC_RETRY_MASK); + if (ret) + return ret; + } + + if (device_property_read_bool(dev, "adi,overvoltage-retry-disable")) { + ret = regmap_clear_bits(st->map, LTC4282_CTRL_LSB, + LTC4282_CTRL_OV_RETRY_MASK); + if (ret) + return ret; + } + + if (device_property_read_bool(dev, "adi,undervoltage-retry-disable")) { + ret = regmap_clear_bits(st->map, LTC4282_CTRL_LSB, + LTC4282_CTRL_UV_RETRY_MASK); + if (ret) + return ret; + } + + if (device_property_read_bool(dev, "adi,fault-log-enable")) { + ret = regmap_set_bits(st->map, LTC4282_ADC_CTRL, + LTC4282_FAULT_LOG_EN_MASK); + if (ret) + return ret; + } + + if (device_property_read_bool(dev, "adi,fault-log-enable")) { + ret = regmap_set_bits(st->map, LTC4282_ADC_CTRL, LTC4282_FAULT_LOG_EN_MASK); + if (ret) + return ret; + } + + ret = device_property_read_u32(dev, "adi,fet-bad-timeout-ms", &val); + if (!ret) { + if (val > LTC4282_FET_BAD_MAX_TIMEOUT) + return dev_err_probe(dev, -EINVAL, + "Invalid value(%u) for adi,fet-bad-timeout-ms", + val); + + ret = regmap_write(st->map, LTC4282_FET_BAD_FAULT_TIMEOUT, val); + if (ret) + return ret; + } + + return ltc4282_gpio_setup(st, dev); +} + +static bool ltc4282_readable_reg(struct device *dev, unsigned int reg) +{ + if (reg == LTC4282_RESERVED_1 || reg == LTC4282_RESERVED_2) + return false; + + return true; +} + +static bool ltc4282_writable_reg(struct device *dev, unsigned int reg) +{ + if (reg == LTC4282_STATUS_LSB || reg == LTC4282_STATUS_MSB) + return false; + if (reg == LTC4282_RESERVED_1 || reg == LTC4282_RESERVED_2) + return false; + + return true; +} + +static const struct regmap_config ltc4282_regmap_config = { + .reg_bits = 8, + .val_bits = 8, + .max_register = LTC4282_RESERVED_3, + .readable_reg = ltc4282_readable_reg, + .writeable_reg = ltc4282_writable_reg, +}; + +static const struct hwmon_channel_info * const ltc4282_info[] = { + HWMON_CHANNEL_INFO(in, + HWMON_I_INPUT | HWMON_I_LOWEST | HWMON_I_HIGHEST | + HWMON_I_MAX | HWMON_I_MIN | HWMON_I_MIN_ALARM | + HWMON_I_MAX_ALARM | HWMON_I_ENABLE | + HWMON_I_RESET_HISTORY | HWMON_I_FAULT | + HWMON_I_LABEL, + HWMON_I_INPUT | HWMON_I_LOWEST | HWMON_I_HIGHEST | + HWMON_I_MAX | HWMON_I_MIN | HWMON_I_MIN_ALARM | + HWMON_I_MAX_ALARM | HWMON_I_LCRIT_ALARM | + HWMON_I_CRIT_ALARM | HWMON_I_ENABLE | + HWMON_I_RESET_HISTORY | HWMON_I_LABEL, + HWMON_I_INPUT | HWMON_I_LOWEST | HWMON_I_HIGHEST | + HWMON_I_MAX | HWMON_I_MIN | HWMON_I_MIN_ALARM | + HWMON_I_RESET_HISTORY | HWMON_I_MAX_ALARM | + HWMON_I_LABEL), + HWMON_CHANNEL_INFO(curr, + HWMON_C_INPUT | HWMON_C_LOWEST | HWMON_C_HIGHEST | + HWMON_C_MAX | HWMON_C_MIN | HWMON_C_MIN_ALARM | + HWMON_C_MAX_ALARM | HWMON_C_CRIT_ALARM | + HWMON_C_RESET_HISTORY | HWMON_C_LABEL), + HWMON_CHANNEL_INFO(power, + HWMON_P_INPUT | HWMON_P_INPUT_LOWEST | + HWMON_P_INPUT_HIGHEST | HWMON_P_MAX | HWMON_P_MIN | + HWMON_P_MAX_ALARM | HWMON_P_MIN_ALARM | + HWMON_P_RESET_HISTORY | HWMON_P_LABEL), + HWMON_CHANNEL_INFO(energy, + HWMON_E_ENABLE), + NULL +}; + +static const struct hwmon_ops ltc4282_hwmon_ops = { + .read = ltc4282_read, + .write = ltc4282_write, + .is_visible = ltc4282_is_visible, + .read_string = ltc4282_read_labels, +}; + +static const struct hwmon_chip_info ltc2947_chip_info = { + .ops = <c4282_hwmon_ops, + .info = ltc4282_info, +}; + +/* energy attributes are 6bytes wide so we need u64 */ +static SENSOR_DEVICE_ATTR_RO(energy1_input, ltc4282_energy, 0); + +static struct attribute *ltc4282_attrs[] = { + &sensor_dev_attr_energy1_input.dev_attr.attr, + NULL +}; +ATTRIBUTE_GROUPS(ltc4282); + +static int ltc4282_show_fault_log(void *arg, u64 *val, u32 mask) +{ + struct ltc4282_state *st = arg; + long alarm; + int ret; + + ret = ltc4282_read_alarm(st, LTC4282_FAULT_LOG, mask, &alarm); + if (ret) + return ret; + + *val = alarm; + + return 0; +} + +static int ltc4282_show_curr1_crit_fault_log(void *arg, u64 *val) +{ + return ltc4282_show_fault_log(arg, val, LTC4282_OC_FAULT_MASK); +} +DEFINE_DEBUGFS_ATTRIBUTE(ltc4282_curr1_crit_fault_log, + ltc4282_show_curr1_crit_fault_log, NULL, "%llu\n"); + +static int ltc4282_show_in1_lcrit_fault_log(void *arg, u64 *val) +{ + return ltc4282_show_fault_log(arg, val, LTC4282_UV_FAULT_MASK); +} +DEFINE_DEBUGFS_ATTRIBUTE(ltc4282_in1_lcrit_fault_log, + ltc4282_show_in1_lcrit_fault_log, NULL, "%llu\n"); + +static int ltc4282_show_in1_crit_fault_log(void *arg, u64 *val) +{ + return ltc4282_show_fault_log(arg, val, LTC4282_OV_FAULT_MASK); +} +DEFINE_DEBUGFS_ATTRIBUTE(ltc4282_in1_crit_fault_log, + ltc4282_show_in1_crit_fault_log, NULL, "%llu\n"); + +static int ltc4282_show_fet_bad_fault_log(void *arg, u64 *val) +{ + return ltc4282_show_fault_log(arg, val, LTC4282_FET_BAD_FAULT_MASK); +} +DEFINE_DEBUGFS_ATTRIBUTE(ltc4282_fet_bad_fault_log, + ltc4282_show_fet_bad_fault_log, NULL, "%llu\n"); + +static int ltc4282_show_fet_short_fault_log(void *arg, u64 *val) +{ + return ltc4282_show_fault_log(arg, val, LTC4282_FET_SHORT_FAULT_MASK); +} +DEFINE_DEBUGFS_ATTRIBUTE(ltc4282_fet_short_fault_log, + ltc4282_show_fet_short_fault_log, NULL, "%llu\n"); + +static int ltc4282_show_power1_bad_fault_log(void *arg, u64 *val) +{ + return ltc4282_show_fault_log(arg, val, LTC4282_POWER_BAD_FAULT_MASK); +} +DEFINE_DEBUGFS_ATTRIBUTE(ltc4282_power1_bad_fault_log, + ltc4282_show_power1_bad_fault_log, NULL, "%llu\n"); + +static void ltc4282_debugfs_remove(void *dir) +{ + debugfs_remove_recursive(dir); +} + +static void ltc4282_debugfs_init(struct ltc4282_state *st, + struct i2c_client *i2c, + const struct device *hwmon) +{ + const char *debugfs_name; + struct dentry *dentry; + int ret; + + if (!IS_ENABLED(CONFIG_DEBUG_FS)) + return; + + debugfs_name = devm_kasprintf(&i2c->dev, GFP_KERNEL, "ltc4282-%s", + dev_name(hwmon)); + if (!debugfs_name) + return; + + dentry = debugfs_create_dir(debugfs_name, NULL); + if (IS_ERR(dentry)) + return; + + ret = devm_add_action_or_reset(&i2c->dev, ltc4282_debugfs_remove, + dentry); + if (ret) + return; + + debugfs_create_file_unsafe("power1_bad_fault_log", 0400, dentry, st, + <c4282_power1_bad_fault_log); + debugfs_create_file_unsafe("in0_fet_short_fault_log", 0400, dentry, st, + <c4282_fet_short_fault_log); + debugfs_create_file_unsafe("in0_fet_bad_fault_log", 0400, dentry, st, + <c4282_fet_bad_fault_log); + debugfs_create_file_unsafe("in1_crit_fault_log", 0400, dentry, st, + <c4282_in1_crit_fault_log); + debugfs_create_file_unsafe("in1_lcrit_fault_log", 0400, dentry, st, + <c4282_in1_lcrit_fault_log); + debugfs_create_file_unsafe("curr1_crit_fault_log", 0400, dentry, st, + <c4282_curr1_crit_fault_log); +} + +static int ltc4282_probe(struct i2c_client *i2c) +{ + struct device *dev = &i2c->dev, *hwmon; + struct ltc4282_state *st; + int ret; + + st = devm_kzalloc(dev, sizeof(*st), GFP_KERNEL); + if (!st) + return dev_err_probe(dev, -ENOMEM, + "Failed to allocate memory\n"); + + st->map = devm_regmap_init_i2c(i2c, <c4282_regmap_config); + if (IS_ERR(st->map)) + return dev_err_probe(dev, PTR_ERR(st->map), + "failed regmap init\n"); + + /* Soft reset */ + ret = regmap_set_bits(st->map, LTC4282_ADC_CTRL, LTC4282_RESET_MASK); + if (ret) + return ret; + + /* Yes, it's big but it is as specified in the datasheet */ + msleep(3200); + + ret = ltc428_clks_setup(st, dev); + if (ret) + return ret; + + ret = ltc4282_setup(st, dev); + if (ret) + return ret; + + mutex_init(&st->lock); + hwmon = devm_hwmon_device_register_with_info(dev, "ltc4282", st, + <c2947_chip_info, + ltc4282_groups); + if (IS_ERR(hwmon)) + return PTR_ERR(hwmon); + + ltc4282_debugfs_init(st, i2c, hwmon); + + return 0; +} + +static const struct of_device_id ltc4282_of_match[] = { + { .compatible = "adi,ltc4282" }, + {} +}; +MODULE_DEVICE_TABLE(of, ltc4282_of_match); + +static struct i2c_driver ltc4282_driver = { + .driver = { + .name = "ltc4282", + .of_match_table = ltc4282_of_match, + }, + .probe = ltc4282_probe, +}; +module_i2c_driver(ltc4282_driver); + +MODULE_AUTHOR("Nuno Sa "); +MODULE_DESCRIPTION("LTC4282 I2C High Current Hot Swap Controller"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hwmon/max127.c b/drivers/hwmon/max127.c index ee5ead06d612..da2289e3560a 100644 --- a/drivers/hwmon/max127.c +++ b/drivers/hwmon/max127.c @@ -335,7 +335,6 @@ static const struct i2c_device_id max127_id[] = { MODULE_DEVICE_TABLE(i2c, max127_id); static struct i2c_driver max127_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "max127", }, diff --git a/drivers/hwmon/max31760.c b/drivers/hwmon/max31760.c index 79945eb466ae..127e31ca3c87 100644 --- a/drivers/hwmon/max31760.c +++ b/drivers/hwmon/max31760.c @@ -60,7 +60,7 @@ static const struct regmap_config regmap_config = { .reg_bits = 8, .val_bits = 8, .max_register = 0x5B, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, .volatile_reg = max31760_volatile_reg, }; @@ -578,7 +578,6 @@ static DEFINE_SIMPLE_DEV_PM_OPS(max31760_pm_ops, max31760_suspend, max31760_resume); static struct i2c_driver max31760_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "max31760", .of_match_table = max31760_of_match, diff --git a/drivers/hwmon/max31790.c b/drivers/hwmon/max31790.c index 0cd44c1e998a..3dc95196b229 100644 --- a/drivers/hwmon/max31790.c +++ b/drivers/hwmon/max31790.c @@ -543,7 +543,6 @@ static const struct i2c_device_id max31790_id[] = { MODULE_DEVICE_TABLE(i2c, max31790_id); static struct i2c_driver max31790_driver = { - .class = I2C_CLASS_HWMON, .probe = max31790_probe, .driver = { .name = "max31790", diff --git a/drivers/hwmon/max31827.c b/drivers/hwmon/max31827.c index 4a8c3e37c5d3..f8a13b30f100 100644 --- a/drivers/hwmon/max31827.c +++ b/drivers/hwmon/max31827.c @@ -652,7 +652,6 @@ static const struct of_device_id max31827_of_match[] = { MODULE_DEVICE_TABLE(of, max31827_of_match); static struct i2c_driver max31827_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "max31827", .of_match_table = max31827_of_match, diff --git a/drivers/hwmon/max6621.c b/drivers/hwmon/max6621.c index af7e62685898..05426cde0e36 100644 --- a/drivers/hwmon/max6621.c +++ b/drivers/hwmon/max6621.c @@ -549,7 +549,6 @@ static const struct of_device_id __maybe_unused max6621_of_match[] = { MODULE_DEVICE_TABLE(of, max6621_of_match); static struct i2c_driver max6621_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = MAX6621_DRV_NAME, .of_match_table = of_match_ptr(max6621_of_match), diff --git a/drivers/hwmon/max6697.c b/drivers/hwmon/max6697.c index 7d10dd434f2e..d161ba0e7813 100644 --- a/drivers/hwmon/max6697.c +++ b/drivers/hwmon/max6697.c @@ -780,7 +780,6 @@ static const struct of_device_id __maybe_unused max6697_of_match[] = { MODULE_DEVICE_TABLE(of, max6697_of_match); static struct i2c_driver max6697_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "max6697", .of_match_table = of_match_ptr(max6697_of_match), diff --git a/drivers/hwmon/nct6683.c b/drivers/hwmon/nct6683.c index 3f3f7a88413e..0d016fedb9c2 100644 --- a/drivers/hwmon/nct6683.c +++ b/drivers/hwmon/nct6683.c @@ -174,6 +174,7 @@ superio_exit(int ioreg) #define NCT6683_CUSTOMER_ID_MITAC 0xa0e #define NCT6683_CUSTOMER_ID_MSI 0x201 #define NCT6683_CUSTOMER_ID_MSI2 0x200 +#define NCT6683_CUSTOMER_ID_MSI3 0x207 #define NCT6683_CUSTOMER_ID_ASROCK 0xe2c #define NCT6683_CUSTOMER_ID_ASROCK2 0xe1b #define NCT6683_CUSTOMER_ID_ASROCK3 0x1631 @@ -1224,6 +1225,8 @@ static int nct6683_probe(struct platform_device *pdev) break; case NCT6683_CUSTOMER_ID_MSI2: break; + case NCT6683_CUSTOMER_ID_MSI3: + break; case NCT6683_CUSTOMER_ID_ASROCK: break; case NCT6683_CUSTOMER_ID_ASROCK2: diff --git a/drivers/hwmon/nct7802.c b/drivers/hwmon/nct7802.c index 024cff151c36..a0e664d5ebfe 100644 --- a/drivers/hwmon/nct7802.c +++ b/drivers/hwmon/nct7802.c @@ -1051,7 +1051,7 @@ static bool nct7802_regmap_is_volatile(struct device *dev, unsigned int reg) static const struct regmap_config nct7802_regmap_config = { .reg_bits = 8, .val_bits = 8, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, .volatile_reg = nct7802_regmap_is_volatile, }; diff --git a/drivers/hwmon/nzxt-kraken3.c b/drivers/hwmon/nzxt-kraken3.c new file mode 100644 index 000000000000..5806a3f32bcb --- /dev/null +++ b/drivers/hwmon/nzxt-kraken3.c @@ -0,0 +1,1008 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * hwmon driver for NZXT Kraken X53/X63/X73 and Z53/Z63/Z73 all in one coolers. + * X53 and Z53 in code refer to all models in their respective series (shortened + * for brevity). + * + * Copyright 2021 Jonas Malaco + * Copyright 2022 Aleksa Savic + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define USB_VENDOR_ID_NZXT 0x1e71 +#define USB_PRODUCT_ID_X53 0x2007 +#define USB_PRODUCT_ID_X53_SECOND 0x2014 +#define USB_PRODUCT_ID_Z53 0x3008 + +enum kinds { X53, Z53 } __packed; +enum pwm_enable { off, manual, curve } __packed; + +static const char *const kraken3_device_names[] = { + [X53] = "x53", + [Z53] = "z53", +}; + +#define DRIVER_NAME "nzxt_kraken3" +#define STATUS_REPORT_ID 0x75 +#define FIRMWARE_REPORT_ID 0x11 +#define STATUS_VALIDITY 2000 /* In ms, equivalent to period of four status reports */ +#define CUSTOM_CURVE_POINTS 40 /* For temps from 20C to 59C (critical temp) */ +#define PUMP_DUTY_MIN 20 /* In percent */ + +/* Sensor report offsets for Kraken X53 and Z53 */ +#define TEMP_SENSOR_START_OFFSET 15 +#define TEMP_SENSOR_END_OFFSET 16 +#define PUMP_SPEED_OFFSET 17 +#define PUMP_DUTY_OFFSET 19 + +/* Firmware version report offset for Kraken X53 and Z53 */ +#define FIRMWARE_VERSION_OFFSET 17 + +/* Sensor report offsets for Kraken Z53 */ +#define Z53_FAN_SPEED_OFFSET 23 +#define Z53_FAN_DUTY_OFFSET 25 + +/* Report offsets for control commands for Kraken X53 and Z53 */ +#define SET_DUTY_ID_OFFSET 1 + +/* Control commands and their lengths for Kraken X53 and Z53 */ + +/* Last byte sets the report interval at 0.5s */ +static const u8 set_interval_cmd[] = { 0x70, 0x02, 0x01, 0xB8, 1 }; +static const u8 finish_init_cmd[] = { 0x70, 0x01 }; +static const u8 __maybe_unused get_fw_version_cmd[] = { 0x10, 0x01 }; +static const u8 set_pump_duty_cmd_header[] = { 0x72, 0x00, 0x00, 0x00 }; +static const u8 z53_get_status_cmd[] = { 0x74, 0x01 }; + +#define SET_INTERVAL_CMD_LENGTH 5 +#define FINISH_INIT_CMD_LENGTH 2 +#define GET_FW_VERSION_CMD_LENGTH 2 +#define MAX_REPORT_LENGTH 64 +#define MIN_REPORT_LENGTH 20 +#define SET_CURVE_DUTY_CMD_HEADER_LENGTH 4 +/* 4 byte header and 40 duty offsets */ +#define SET_CURVE_DUTY_CMD_LENGTH (4 + 40) +#define Z53_GET_STATUS_CMD_LENGTH 2 + +static const char *const kraken3_temp_label[] = { + "Coolant temp", +}; + +static const char *const kraken3_fan_label[] = { + "Pump speed", + "Fan speed" +}; + +struct kraken3_channel_info { + enum pwm_enable mode; + + /* Both values are PWM */ + u16 reported_duty; + u16 fixed_duty; /* Manually set fixed duty */ + + u8 pwm_points[CUSTOM_CURVE_POINTS]; +}; + +struct kraken3_data { + struct hid_device *hdev; + struct device *hwmon_dev; + struct dentry *debugfs; + struct mutex buffer_lock; /* For locking access to buffer */ + struct mutex z53_status_request_lock; + struct completion fw_version_processed; + /* + * For X53 devices, tracks whether an initial (one) sensor report was received to + * make fancontrol not bail outright. For Z53 devices, whether a status report + * was processed after requesting one. + */ + struct completion status_report_processed; + /* For locking the above completion */ + spinlock_t status_completion_lock; + + u8 *buffer; + struct kraken3_channel_info channel_info[2]; /* Pump and fan */ + bool is_device_faulty; + + /* Sensor values */ + s32 temp_input[1]; + u16 fan_input[2]; + + enum kinds kind; + u8 firmware_version[3]; + + unsigned long updated; /* jiffies */ +}; + +static umode_t kraken3_is_visible(const void *data, enum hwmon_sensor_types type, u32 attr, + int channel) +{ + const struct kraken3_data *priv = data; + + switch (type) { + case hwmon_temp: + if (channel < 1) + return 0444; + break; + case hwmon_fan: + switch (priv->kind) { + case X53: + /* Just the pump */ + if (channel < 1) + return 0444; + break; + case Z53: + /* Pump and fan */ + if (channel < 2) + return 0444; + break; + default: + break; + } + break; + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_enable: + case hwmon_pwm_input: + switch (priv->kind) { + case X53: + /* Just the pump */ + if (channel < 1) + return 0644; + break; + case Z53: + /* Pump and fan */ + if (channel < 2) + return 0644; + break; + default: + break; + } + break; + default: + break; + } + break; + default: + break; + } + + return 0; +} + +/* + * Writes the command to the device with the rest of the report (up to 64 bytes) filled + * with zeroes. + */ +static int kraken3_write_expanded(struct kraken3_data *priv, const u8 *cmd, int cmd_length) +{ + int ret; + + mutex_lock(&priv->buffer_lock); + + memcpy_and_pad(priv->buffer, MAX_REPORT_LENGTH, cmd, cmd_length, 0x00); + ret = hid_hw_output_report(priv->hdev, priv->buffer, MAX_REPORT_LENGTH); + + mutex_unlock(&priv->buffer_lock); + return ret; +} + +static int kraken3_percent_to_pwm(long val) +{ + return DIV_ROUND_CLOSEST(val * 255, 100); +} + +static int kraken3_pwm_to_percent(long val, int channel) +{ + int percent_value; + + if (val < 0 || val > 255) + return -EINVAL; + + percent_value = DIV_ROUND_CLOSEST(val * 100, 255); + + /* Bring up pump duty to min value if needed */ + if (channel == 0 && percent_value < PUMP_DUTY_MIN) + percent_value = PUMP_DUTY_MIN; + + return percent_value; +} + +static int kraken3_read_x53(struct kraken3_data *priv) +{ + int ret; + + if (completion_done(&priv->status_report_processed)) + /* + * We're here because data is stale. This means that sensor reports haven't + * been received for some time in kraken3_raw_event(). On X-series sensor data + * can't be manually requested, so return an error. + */ + return -ENODATA; + + /* + * Data needs to be read, but a sensor report wasn't yet received. It's usually + * fancontrol that requests data this early and it exits if it reads an error code. + * So, wait for the first report to be parsed (but up to STATUS_VALIDITY). + * This does not concern the Z series devices, because they send a sensor report + * only when requested. + */ + ret = wait_for_completion_interruptible_timeout(&priv->status_report_processed, + msecs_to_jiffies(STATUS_VALIDITY)); + if (ret == 0) + return -ETIMEDOUT; + else if (ret < 0) + return ret; + + /* The first sensor report was parsed on time and reading can continue */ + return 0; +} + +static int kraken3_read_z53(struct kraken3_data *priv) +{ + int ret = mutex_lock_interruptible(&priv->z53_status_request_lock); + + if (ret < 0) + return ret; + + if (!time_after(jiffies, priv->updated + msecs_to_jiffies(STATUS_VALIDITY))) { + /* Data is up to date */ + goto unlock_and_return; + } + + /* + * Disable interrupts for a moment to safely reinit the completion, + * as hidraw calls could have allowed one or more readers to complete. + */ + spin_lock_bh(&priv->status_completion_lock); + reinit_completion(&priv->status_report_processed); + spin_unlock_bh(&priv->status_completion_lock); + + /* Send command for getting status */ + ret = kraken3_write_expanded(priv, z53_get_status_cmd, Z53_GET_STATUS_CMD_LENGTH); + if (ret < 0) + goto unlock_and_return; + + /* Wait for completion from kraken3_raw_event() */ + ret = wait_for_completion_interruptible_timeout(&priv->status_report_processed, + msecs_to_jiffies(STATUS_VALIDITY)); + if (ret == 0) + ret = -ETIMEDOUT; + +unlock_and_return: + mutex_unlock(&priv->z53_status_request_lock); + if (ret < 0) + return ret; + + return 0; +} + +static int kraken3_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, + long *val) +{ + struct kraken3_data *priv = dev_get_drvdata(dev); + int ret; + + if (time_after(jiffies, priv->updated + msecs_to_jiffies(STATUS_VALIDITY))) { + if (priv->kind == X53) + ret = kraken3_read_x53(priv); + else + ret = kraken3_read_z53(priv); + + if (ret < 0) + return ret; + + if (priv->is_device_faulty) + return -ENODATA; + } + + switch (type) { + case hwmon_temp: + *val = priv->temp_input[channel]; + break; + case hwmon_fan: + *val = priv->fan_input[channel]; + break; + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_enable: + *val = priv->channel_info[channel].mode; + break; + case hwmon_pwm_input: + *val = priv->channel_info[channel].reported_duty; + break; + default: + return -EOPNOTSUPP; + } + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + +static int kraken3_read_string(struct device *dev, enum hwmon_sensor_types type, u32 attr, + int channel, const char **str) +{ + switch (type) { + case hwmon_temp: + *str = kraken3_temp_label[channel]; + break; + case hwmon_fan: + *str = kraken3_fan_label[channel]; + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + +/* Writes custom curve to device */ +static int kraken3_write_curve(struct kraken3_data *priv, u8 *curve_array, int channel) +{ + u8 fixed_duty_cmd[SET_CURVE_DUTY_CMD_LENGTH]; + int ret; + + /* Copy command header */ + memcpy(fixed_duty_cmd, set_pump_duty_cmd_header, SET_CURVE_DUTY_CMD_HEADER_LENGTH); + + /* Set the correct ID for writing pump/fan duty (0x01 or 0x02, respectively) */ + fixed_duty_cmd[SET_DUTY_ID_OFFSET] = channel + 1; + + /* Copy curve to command */ + memcpy(fixed_duty_cmd + SET_CURVE_DUTY_CMD_HEADER_LENGTH, curve_array, CUSTOM_CURVE_POINTS); + + ret = kraken3_write_expanded(priv, fixed_duty_cmd, SET_CURVE_DUTY_CMD_LENGTH); + return ret; +} + +static int kraken3_write_fixed_duty(struct kraken3_data *priv, long val, int channel) +{ + u8 fixed_curve_points[CUSTOM_CURVE_POINTS]; + int ret, percent_val, i; + + percent_val = kraken3_pwm_to_percent(val, channel); + if (percent_val < 0) + return percent_val; + + /* + * The devices can only control the duty through a curve. + * Since we're setting a fixed duty here, fill the whole curve + * (ranging from 20C to 59C) with the same duty, except for + * the last point, the critical temperature, where it's maxed + * out for safety. + */ + + /* Fill the custom curve with the fixed value we're setting */ + for (i = 0; i < CUSTOM_CURVE_POINTS - 1; i++) + fixed_curve_points[i] = percent_val; + + /* Force duty to 100% at critical temp */ + fixed_curve_points[CUSTOM_CURVE_POINTS - 1] = 100; + + /* Write the fixed duty curve to the device */ + ret = kraken3_write_curve(priv, fixed_curve_points, channel); + return ret; +} + +static int kraken3_write(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, + long val) +{ + struct kraken3_data *priv = dev_get_drvdata(dev); + int ret; + + switch (type) { + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_input: + /* Remember the last set fixed duty for channel */ + priv->channel_info[channel].fixed_duty = val; + + if (priv->channel_info[channel].mode == manual) { + ret = kraken3_write_fixed_duty(priv, val, channel); + if (ret < 0) + return ret; + + /* + * Lock onto this value and report it until next interrupt status + * report is received, so userspace tools can continue to work. + */ + priv->channel_info[channel].reported_duty = val; + } + break; + case hwmon_pwm_enable: + if (val < 0 || val > 2) + return -EINVAL; + + switch (val) { + case 0: + /* Set channel to 100%, direct duty value */ + ret = kraken3_write_fixed_duty(priv, 255, channel); + if (ret < 0) + return ret; + + /* We don't control anything anymore */ + priv->channel_info[channel].mode = off; + break; + case 1: + /* Apply the last known direct duty value */ + ret = + kraken3_write_fixed_duty(priv, + priv->channel_info[channel].fixed_duty, + channel); + if (ret < 0) + return ret; + + priv->channel_info[channel].mode = manual; + break; + case 2: + /* Apply the curve and note as enabled */ + ret = + kraken3_write_curve(priv, + priv->channel_info[channel].pwm_points, + channel); + if (ret < 0) + return ret; + + priv->channel_info[channel].mode = curve; + break; + default: + break; + } + break; + default: + return -EOPNOTSUPP; + } + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + +static ssize_t kraken3_fan_curve_pwm_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct sensor_device_attribute_2 *dev_attr = to_sensor_dev_attr_2(attr); + struct kraken3_data *priv = dev_get_drvdata(dev); + long val; + int ret; + + if (kstrtol(buf, 10, &val) < 0) + return -EINVAL; + + val = kraken3_pwm_to_percent(val, dev_attr->nr); + if (val < 0) + return val; + + priv->channel_info[dev_attr->nr].pwm_points[dev_attr->index] = val; + + if (priv->channel_info[dev_attr->nr].mode == curve) { + /* Apply the curve */ + ret = + kraken3_write_curve(priv, + priv->channel_info[dev_attr->nr].pwm_points, dev_attr->nr); + if (ret < 0) + return ret; + } + + return count; +} + +static umode_t kraken3_curve_props_are_visible(struct kobject *kobj, struct attribute *attr, + int index) +{ + struct device *dev = kobj_to_dev(kobj); + struct kraken3_data *priv = dev_get_drvdata(dev); + + /* Only Z53 has the fan curve */ + if (index >= CUSTOM_CURVE_POINTS && priv->kind != Z53) + return 0; + + return attr->mode; +} + +/* Custom pump curve from 20C to 59C (critical temp) */ +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point1_pwm, kraken3_fan_curve_pwm, 0, 0); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point2_pwm, kraken3_fan_curve_pwm, 0, 1); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point3_pwm, kraken3_fan_curve_pwm, 0, 2); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point4_pwm, kraken3_fan_curve_pwm, 0, 3); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point5_pwm, kraken3_fan_curve_pwm, 0, 4); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point6_pwm, kraken3_fan_curve_pwm, 0, 5); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point7_pwm, kraken3_fan_curve_pwm, 0, 6); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point8_pwm, kraken3_fan_curve_pwm, 0, 7); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point9_pwm, kraken3_fan_curve_pwm, 0, 8); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point10_pwm, kraken3_fan_curve_pwm, 0, 9); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point11_pwm, kraken3_fan_curve_pwm, 0, 10); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point12_pwm, kraken3_fan_curve_pwm, 0, 11); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point13_pwm, kraken3_fan_curve_pwm, 0, 12); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point14_pwm, kraken3_fan_curve_pwm, 0, 13); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point15_pwm, kraken3_fan_curve_pwm, 0, 14); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point16_pwm, kraken3_fan_curve_pwm, 0, 15); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point17_pwm, kraken3_fan_curve_pwm, 0, 16); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point18_pwm, kraken3_fan_curve_pwm, 0, 17); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point19_pwm, kraken3_fan_curve_pwm, 0, 18); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point20_pwm, kraken3_fan_curve_pwm, 0, 19); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point21_pwm, kraken3_fan_curve_pwm, 0, 20); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point22_pwm, kraken3_fan_curve_pwm, 0, 21); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point23_pwm, kraken3_fan_curve_pwm, 0, 22); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point24_pwm, kraken3_fan_curve_pwm, 0, 23); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point25_pwm, kraken3_fan_curve_pwm, 0, 24); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point26_pwm, kraken3_fan_curve_pwm, 0, 25); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point27_pwm, kraken3_fan_curve_pwm, 0, 26); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point28_pwm, kraken3_fan_curve_pwm, 0, 27); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point29_pwm, kraken3_fan_curve_pwm, 0, 28); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point30_pwm, kraken3_fan_curve_pwm, 0, 29); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point31_pwm, kraken3_fan_curve_pwm, 0, 30); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point32_pwm, kraken3_fan_curve_pwm, 0, 31); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point33_pwm, kraken3_fan_curve_pwm, 0, 32); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point34_pwm, kraken3_fan_curve_pwm, 0, 33); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point35_pwm, kraken3_fan_curve_pwm, 0, 34); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point36_pwm, kraken3_fan_curve_pwm, 0, 35); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point37_pwm, kraken3_fan_curve_pwm, 0, 36); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point38_pwm, kraken3_fan_curve_pwm, 0, 37); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point39_pwm, kraken3_fan_curve_pwm, 0, 38); +static SENSOR_DEVICE_ATTR_2_WO(temp1_auto_point40_pwm, kraken3_fan_curve_pwm, 0, 39); + +/* Custom fan curve from 20C to 59C (critical temp) */ +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point1_pwm, kraken3_fan_curve_pwm, 1, 0); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point2_pwm, kraken3_fan_curve_pwm, 1, 1); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point3_pwm, kraken3_fan_curve_pwm, 1, 2); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point4_pwm, kraken3_fan_curve_pwm, 1, 3); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point5_pwm, kraken3_fan_curve_pwm, 1, 4); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point6_pwm, kraken3_fan_curve_pwm, 1, 5); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point7_pwm, kraken3_fan_curve_pwm, 1, 6); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point8_pwm, kraken3_fan_curve_pwm, 1, 7); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point9_pwm, kraken3_fan_curve_pwm, 1, 8); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point10_pwm, kraken3_fan_curve_pwm, 1, 9); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point11_pwm, kraken3_fan_curve_pwm, 1, 10); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point12_pwm, kraken3_fan_curve_pwm, 1, 11); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point13_pwm, kraken3_fan_curve_pwm, 1, 12); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point14_pwm, kraken3_fan_curve_pwm, 1, 13); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point15_pwm, kraken3_fan_curve_pwm, 1, 14); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point16_pwm, kraken3_fan_curve_pwm, 1, 15); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point17_pwm, kraken3_fan_curve_pwm, 1, 16); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point18_pwm, kraken3_fan_curve_pwm, 1, 17); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point19_pwm, kraken3_fan_curve_pwm, 1, 18); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point20_pwm, kraken3_fan_curve_pwm, 1, 19); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point21_pwm, kraken3_fan_curve_pwm, 1, 20); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point22_pwm, kraken3_fan_curve_pwm, 1, 21); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point23_pwm, kraken3_fan_curve_pwm, 1, 22); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point24_pwm, kraken3_fan_curve_pwm, 1, 23); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point25_pwm, kraken3_fan_curve_pwm, 1, 24); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point26_pwm, kraken3_fan_curve_pwm, 1, 25); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point27_pwm, kraken3_fan_curve_pwm, 1, 26); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point28_pwm, kraken3_fan_curve_pwm, 1, 27); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point29_pwm, kraken3_fan_curve_pwm, 1, 28); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point30_pwm, kraken3_fan_curve_pwm, 1, 29); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point31_pwm, kraken3_fan_curve_pwm, 1, 30); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point32_pwm, kraken3_fan_curve_pwm, 1, 31); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point33_pwm, kraken3_fan_curve_pwm, 1, 32); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point34_pwm, kraken3_fan_curve_pwm, 1, 33); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point35_pwm, kraken3_fan_curve_pwm, 1, 34); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point36_pwm, kraken3_fan_curve_pwm, 1, 35); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point37_pwm, kraken3_fan_curve_pwm, 1, 36); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point38_pwm, kraken3_fan_curve_pwm, 1, 37); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point39_pwm, kraken3_fan_curve_pwm, 1, 38); +static SENSOR_DEVICE_ATTR_2_WO(temp2_auto_point40_pwm, kraken3_fan_curve_pwm, 1, 39); + +static struct attribute *kraken3_curve_attrs[] = { + /* Pump control curve */ + &sensor_dev_attr_temp1_auto_point1_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point2_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point3_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point4_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point5_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point6_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point7_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point8_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point9_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point10_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point11_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point12_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point13_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point14_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point15_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point16_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point17_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point18_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point19_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point20_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point21_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point22_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point23_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point24_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point25_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point26_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point27_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point28_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point29_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point30_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point31_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point32_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point33_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point34_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point35_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point36_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point37_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point38_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point39_pwm.dev_attr.attr, + &sensor_dev_attr_temp1_auto_point40_pwm.dev_attr.attr, + /* Fan control curve (Z53 only) */ + &sensor_dev_attr_temp2_auto_point1_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point2_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point3_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point4_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point5_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point6_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point7_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point8_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point9_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point10_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point11_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point12_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point13_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point14_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point15_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point16_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point17_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point18_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point19_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point20_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point21_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point22_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point23_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point24_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point25_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point26_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point27_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point28_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point29_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point30_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point31_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point32_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point33_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point34_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point35_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point36_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point37_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point38_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point39_pwm.dev_attr.attr, + &sensor_dev_attr_temp2_auto_point40_pwm.dev_attr.attr, + NULL +}; + +static const struct attribute_group kraken3_curves_group = { + .attrs = kraken3_curve_attrs, + .is_visible = kraken3_curve_props_are_visible +}; + +static const struct attribute_group *kraken3_groups[] = { + &kraken3_curves_group, + NULL +}; + +static const struct hwmon_ops kraken3_hwmon_ops = { + .is_visible = kraken3_is_visible, + .read = kraken3_read, + .read_string = kraken3_read_string, + .write = kraken3_write +}; + +static const struct hwmon_channel_info *kraken3_info[] = { + HWMON_CHANNEL_INFO(temp, + HWMON_T_INPUT | HWMON_T_LABEL), + HWMON_CHANNEL_INFO(fan, + HWMON_F_INPUT | HWMON_F_LABEL, + HWMON_F_INPUT | HWMON_F_LABEL, + HWMON_F_INPUT | HWMON_F_LABEL, + HWMON_F_INPUT | HWMON_F_LABEL), + HWMON_CHANNEL_INFO(pwm, + HWMON_PWM_INPUT | HWMON_PWM_ENABLE, + HWMON_PWM_INPUT | HWMON_PWM_ENABLE), + NULL +}; + +static const struct hwmon_chip_info kraken3_chip_info = { + .ops = &kraken3_hwmon_ops, + .info = kraken3_info, +}; + +static int kraken3_raw_event(struct hid_device *hdev, struct hid_report *report, u8 *data, int size) +{ + struct kraken3_data *priv = hid_get_drvdata(hdev); + int i; + + if (size < MIN_REPORT_LENGTH) + return 0; + + if (report->id == FIRMWARE_REPORT_ID) { + /* Read firmware version */ + for (i = 0; i < 3; i++) + priv->firmware_version[i] = data[FIRMWARE_VERSION_OFFSET + i]; + + if (!completion_done(&priv->fw_version_processed)) + complete_all(&priv->fw_version_processed); + + return 0; + } + + if (report->id != STATUS_REPORT_ID) + return 0; + + if (data[TEMP_SENSOR_START_OFFSET] == 0xff && data[TEMP_SENSOR_END_OFFSET] == 0xff) { + hid_err_once(hdev, + "firmware or device is possibly damaged (is SATA power connected?), not parsing reports\n"); + + /* + * Mark first X-series device report as received, + * as well as all for Z-series, if faulty. + */ + spin_lock(&priv->status_completion_lock); + if (priv->kind != X53 || !completion_done(&priv->status_report_processed)) { + priv->is_device_faulty = true; + complete_all(&priv->status_report_processed); + } + spin_unlock(&priv->status_completion_lock); + + return 0; + } + + /* Received normal data */ + priv->is_device_faulty = false; + + /* Temperature and fan sensor readings */ + priv->temp_input[0] = + data[TEMP_SENSOR_START_OFFSET] * 1000 + data[TEMP_SENSOR_END_OFFSET] * 100; + + priv->fan_input[0] = get_unaligned_le16(data + PUMP_SPEED_OFFSET); + priv->channel_info[0].reported_duty = kraken3_percent_to_pwm(data[PUMP_DUTY_OFFSET]); + + spin_lock(&priv->status_completion_lock); + if (priv->kind == X53 && !completion_done(&priv->status_report_processed)) { + /* Mark first X-series device report as received */ + complete_all(&priv->status_report_processed); + } else if (priv->kind == Z53) { + /* Additional readings for Z53 */ + priv->fan_input[1] = get_unaligned_le16(data + Z53_FAN_SPEED_OFFSET); + priv->channel_info[1].reported_duty = + kraken3_percent_to_pwm(data[Z53_FAN_DUTY_OFFSET]); + + if (!completion_done(&priv->status_report_processed)) + complete_all(&priv->status_report_processed); + } + spin_unlock(&priv->status_completion_lock); + + priv->updated = jiffies; + + return 0; +} + +static int kraken3_init_device(struct hid_device *hdev) +{ + struct kraken3_data *priv = hid_get_drvdata(hdev); + int ret; + + /* Set the polling interval */ + ret = kraken3_write_expanded(priv, set_interval_cmd, SET_INTERVAL_CMD_LENGTH); + if (ret < 0) + return ret; + + /* Finalize the init process */ + ret = kraken3_write_expanded(priv, finish_init_cmd, FINISH_INIT_CMD_LENGTH); + if (ret < 0) + return ret; + + return 0; +} + +static int kraken3_get_fw_ver(struct hid_device *hdev) +{ + struct kraken3_data *priv = hid_get_drvdata(hdev); + int ret; + + ret = kraken3_write_expanded(priv, get_fw_version_cmd, GET_FW_VERSION_CMD_LENGTH); + if (ret < 0) + return ret; + + ret = wait_for_completion_interruptible_timeout(&priv->fw_version_processed, + msecs_to_jiffies(STATUS_VALIDITY)); + if (ret == 0) + return -ETIMEDOUT; + else if (ret < 0) + return ret; + + return 0; +} + +static int __maybe_unused kraken3_reset_resume(struct hid_device *hdev) +{ + int ret; + + ret = kraken3_init_device(hdev); + if (ret) + hid_err(hdev, "req init (reset_resume) failed with %d\n", ret); + + return ret; +} + +static int firmware_version_show(struct seq_file *seqf, void *unused) +{ + struct kraken3_data *priv = seqf->private; + + seq_printf(seqf, "%u.%u.%u\n", priv->firmware_version[0], priv->firmware_version[1], + priv->firmware_version[2]); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(firmware_version); + +static void kraken3_debugfs_init(struct kraken3_data *priv) +{ + char name[64]; + + if (!priv->firmware_version[0]) + return; /* Nothing to display in debugfs */ + + scnprintf(name, sizeof(name), "%s_%s-%s", DRIVER_NAME, kraken3_device_names[priv->kind], + dev_name(&priv->hdev->dev)); + + priv->debugfs = debugfs_create_dir(name, NULL); + debugfs_create_file("firmware_version", 0444, priv->debugfs, priv, &firmware_version_fops); +} + +static int kraken3_probe(struct hid_device *hdev, const struct hid_device_id *id) +{ + struct kraken3_data *priv; + int ret; + + priv = devm_kzalloc(&hdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->hdev = hdev; + hid_set_drvdata(hdev, priv); + + /* + * Initialize ->updated to STATUS_VALIDITY seconds in the past, making + * the initial empty data invalid for kraken3_read without the need for + * a special case there. + */ + priv->updated = jiffies - msecs_to_jiffies(STATUS_VALIDITY); + + ret = hid_parse(hdev); + if (ret) { + hid_err(hdev, "hid parse failed with %d\n", ret); + return ret; + } + + /* Enable hidraw so existing user-space tools can continue to work */ + ret = hid_hw_start(hdev, HID_CONNECT_HIDRAW); + if (ret) { + hid_err(hdev, "hid hw start failed with %d\n", ret); + return ret; + } + + ret = hid_hw_open(hdev); + if (ret) { + hid_err(hdev, "hid hw open failed with %d\n", ret); + goto fail_and_stop; + } + + switch (hdev->product) { + case USB_PRODUCT_ID_X53: + case USB_PRODUCT_ID_X53_SECOND: + priv->kind = X53; + break; + case USB_PRODUCT_ID_Z53: + priv->kind = Z53; + break; + default: + break; + } + + priv->buffer = devm_kzalloc(&hdev->dev, MAX_REPORT_LENGTH, GFP_KERNEL); + if (!priv->buffer) { + ret = -ENOMEM; + goto fail_and_close; + } + + mutex_init(&priv->buffer_lock); + mutex_init(&priv->z53_status_request_lock); + init_completion(&priv->fw_version_processed); + init_completion(&priv->status_report_processed); + spin_lock_init(&priv->status_completion_lock); + + hid_device_io_start(hdev); + ret = kraken3_init_device(hdev); + if (ret < 0) { + hid_err(hdev, "device init failed with %d\n", ret); + goto fail_and_close; + } + + ret = kraken3_get_fw_ver(hdev); + if (ret < 0) + hid_warn(hdev, "fw version request failed with %d\n", ret); + + priv->hwmon_dev = hwmon_device_register_with_info(&hdev->dev, + kraken3_device_names[priv->kind], priv, + &kraken3_chip_info, kraken3_groups); + if (IS_ERR(priv->hwmon_dev)) { + ret = PTR_ERR(priv->hwmon_dev); + hid_err(hdev, "hwmon registration failed with %d\n", ret); + goto fail_and_close; + } + + kraken3_debugfs_init(priv); + + return 0; + +fail_and_close: + hid_hw_close(hdev); +fail_and_stop: + hid_hw_stop(hdev); + return ret; +} + +static void kraken3_remove(struct hid_device *hdev) +{ + struct kraken3_data *priv = hid_get_drvdata(hdev); + + debugfs_remove_recursive(priv->debugfs); + hwmon_device_unregister(priv->hwmon_dev); + + hid_hw_close(hdev); + hid_hw_stop(hdev); +} + +static const struct hid_device_id kraken3_table[] = { + /* NZXT Kraken X53/X63/X73 have two possible product IDs */ + { HID_USB_DEVICE(USB_VENDOR_ID_NZXT, USB_PRODUCT_ID_X53) }, + { HID_USB_DEVICE(USB_VENDOR_ID_NZXT, USB_PRODUCT_ID_X53_SECOND) }, + { HID_USB_DEVICE(USB_VENDOR_ID_NZXT, USB_PRODUCT_ID_Z53) }, + { } +}; + +MODULE_DEVICE_TABLE(hid, kraken3_table); + +static struct hid_driver kraken3_driver = { + .name = DRIVER_NAME, + .id_table = kraken3_table, + .probe = kraken3_probe, + .remove = kraken3_remove, + .raw_event = kraken3_raw_event, +#ifdef CONFIG_PM + .reset_resume = kraken3_reset_resume, +#endif +}; + +static int __init kraken3_init(void) +{ + return hid_register_driver(&kraken3_driver); +} + +static void __exit kraken3_exit(void) +{ + hid_unregister_driver(&kraken3_driver); +} + +/* When compiled into the kernel, initialize after the HID bus */ +late_initcall(kraken3_init); +module_exit(kraken3_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Jonas Malaco "); +MODULE_AUTHOR("Aleksa Savic "); +MODULE_DESCRIPTION("Hwmon driver for NZXT Kraken X53/X63/X73, Z53/Z63/Z73 coolers"); diff --git a/drivers/hwmon/occ/p8_i2c.c b/drivers/hwmon/occ/p8_i2c.c index 06095975f5c8..31159606cec7 100644 --- a/drivers/hwmon/occ/p8_i2c.c +++ b/drivers/hwmon/occ/p8_i2c.c @@ -241,7 +241,6 @@ static const struct of_device_id p8_i2c_occ_of_match[] = { MODULE_DEVICE_TABLE(of, p8_i2c_occ_of_match); static struct i2c_driver p8_i2c_occ_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "occ-hwmon", .of_match_table = p8_i2c_occ_of_match, diff --git a/drivers/hwmon/oxp-sensors.c b/drivers/hwmon/oxp-sensors.c index ea9602063eab..8d3b0f86cc57 100644 --- a/drivers/hwmon/oxp-sensors.c +++ b/drivers/hwmon/oxp-sensors.c @@ -43,6 +43,7 @@ enum oxp_board { aok_zoe_a1 = 1, aya_neo_2, aya_neo_air, + aya_neo_air_plus_mendo, aya_neo_air_pro, aya_neo_geek, oxp_mini_amd, @@ -98,6 +99,13 @@ static const struct dmi_system_id dmi_table[] = { }, .driver_data = (void *)aya_neo_air, }, + { + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "AYANEO"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "AB05-Mendocino"), + }, + .driver_data = (void *)aya_neo_air_plus_mendo, + }, { .matches = { DMI_MATCH(DMI_BOARD_VENDOR, "AYANEO"), @@ -332,6 +340,7 @@ static int oxp_platform_read(struct device *dev, enum hwmon_sensor_types type, switch (board) { case aya_neo_2: case aya_neo_air: + case aya_neo_air_plus_mendo: case aya_neo_air_pro: case aya_neo_geek: case oxp_mini_amd: @@ -374,6 +383,7 @@ static int oxp_platform_write(struct device *dev, enum hwmon_sensor_types type, switch (board) { case aya_neo_2: case aya_neo_air: + case aya_neo_air_plus_mendo: case aya_neo_air_pro: case aya_neo_geek: case oxp_mini_amd: diff --git a/drivers/hwmon/pmbus/Kconfig b/drivers/hwmon/pmbus/Kconfig index 294808f5240a..557ae0c414b0 100644 --- a/drivers/hwmon/pmbus/Kconfig +++ b/drivers/hwmon/pmbus/Kconfig @@ -377,6 +377,15 @@ config SENSORS_MPQ7932 This driver can also be built as a module. If so, the module will be called mpq7932. +config SENSORS_MPQ8785 + tristate "MPS MPQ8785" + help + If you say yes here you get hardware monitoring functionality support + for power management IC MPS MPQ8785. + + This driver can also be built as a module. If so, the module will + be called mpq8785. + config SENSORS_PIM4328 tristate "Flex PIM4328 and compatibles" help diff --git a/drivers/hwmon/pmbus/Makefile b/drivers/hwmon/pmbus/Makefile index cf8a76744545..f14ecf03ad77 100644 --- a/drivers/hwmon/pmbus/Makefile +++ b/drivers/hwmon/pmbus/Makefile @@ -39,6 +39,7 @@ obj-$(CONFIG_SENSORS_MP2975) += mp2975.o obj-$(CONFIG_SENSORS_MP5023) += mp5023.o obj-$(CONFIG_SENSORS_MP5990) += mp5990.o obj-$(CONFIG_SENSORS_MPQ7932) += mpq7932.o +obj-$(CONFIG_SENSORS_MPQ8785) += mpq8785.o obj-$(CONFIG_SENSORS_PLI1209BC) += pli1209bc.o obj-$(CONFIG_SENSORS_PM6764TR) += pm6764tr.o obj-$(CONFIG_SENSORS_PXE1610) += pxe1610.o diff --git a/drivers/hwmon/pmbus/ir36021.c b/drivers/hwmon/pmbus/ir36021.c index 382ba6b6031a..a263afeb8ac1 100644 --- a/drivers/hwmon/pmbus/ir36021.c +++ b/drivers/hwmon/pmbus/ir36021.c @@ -63,7 +63,6 @@ static const struct of_device_id __maybe_unused ir36021_of_id[] = { MODULE_DEVICE_TABLE(of, ir36021_of_id); static struct i2c_driver ir36021_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "ir36021", .of_match_table = of_match_ptr(ir36021_of_id), diff --git a/drivers/hwmon/pmbus/ir38064.c b/drivers/hwmon/pmbus/ir38064.c index 04185be3fdb6..69e18cb468f6 100644 --- a/drivers/hwmon/pmbus/ir38064.c +++ b/drivers/hwmon/pmbus/ir38064.c @@ -22,7 +22,7 @@ #if IS_ENABLED(CONFIG_SENSORS_IR38064_REGULATOR) static const struct regulator_desc ir38064_reg_desc[] = { - PMBUS_REGULATOR("vout", 0), + PMBUS_REGULATOR_ONE("vout"), }; #endif /* CONFIG_SENSORS_IR38064_REGULATOR */ diff --git a/drivers/hwmon/pmbus/lm25066.c b/drivers/hwmon/pmbus/lm25066.c index 3a20df5a43ec..cfffa4cdc0df 100644 --- a/drivers/hwmon/pmbus/lm25066.c +++ b/drivers/hwmon/pmbus/lm25066.c @@ -437,7 +437,7 @@ static int lm25066_write_word_data(struct i2c_client *client, int page, int reg, #if IS_ENABLED(CONFIG_SENSORS_LM25066_REGULATOR) static const struct regulator_desc lm25066_reg_desc[] = { - PMBUS_REGULATOR("vout", 0), + PMBUS_REGULATOR_ONE("vout"), }; #endif diff --git a/drivers/hwmon/pmbus/mpq8785.c b/drivers/hwmon/pmbus/mpq8785.c new file mode 100644 index 000000000000..4e2549cc8120 --- /dev/null +++ b/drivers/hwmon/pmbus/mpq8785.c @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Driver for MPS MPQ8785 Step-Down Converter + */ + +#include +#include +#include +#include "pmbus.h" + +static int mpq8785_identify(struct i2c_client *client, + struct pmbus_driver_info *info) +{ + int vout_mode; + + vout_mode = pmbus_read_byte_data(client, 0, PMBUS_VOUT_MODE); + if (vout_mode < 0 || vout_mode == 0xff) + return vout_mode < 0 ? vout_mode : -ENODEV; + switch (vout_mode >> 5) { + case 0: + info->format[PSC_VOLTAGE_OUT] = linear; + break; + case 1: + case 2: + info->format[PSC_VOLTAGE_OUT] = direct, + info->m[PSC_VOLTAGE_OUT] = 64; + info->b[PSC_VOLTAGE_OUT] = 0; + info->R[PSC_VOLTAGE_OUT] = 1; + break; + default: + return -ENODEV; + } + + return 0; +}; + +static struct pmbus_driver_info mpq8785_info = { + .pages = 1, + .format[PSC_VOLTAGE_IN] = direct, + .format[PSC_CURRENT_OUT] = direct, + .format[PSC_TEMPERATURE] = direct, + .m[PSC_VOLTAGE_IN] = 4, + .b[PSC_VOLTAGE_IN] = 0, + .R[PSC_VOLTAGE_IN] = 1, + .m[PSC_CURRENT_OUT] = 16, + .b[PSC_CURRENT_OUT] = 0, + .R[PSC_CURRENT_OUT] = 0, + .m[PSC_TEMPERATURE] = 1, + .b[PSC_TEMPERATURE] = 0, + .R[PSC_TEMPERATURE] = 0, + .func[0] = + PMBUS_HAVE_VIN | PMBUS_HAVE_STATUS_INPUT | + PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT | + PMBUS_HAVE_IOUT | PMBUS_HAVE_STATUS_IOUT | + PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP, + .identify = mpq8785_identify, +}; + +static int mpq8785_probe(struct i2c_client *client) +{ + return pmbus_do_probe(client, &mpq8785_info); +}; + +static const struct i2c_device_id mpq8785_id[] = { + { "mpq8785", 0 }, + { }, +}; +MODULE_DEVICE_TABLE(i2c, mpq8785_id); + +static const struct of_device_id __maybe_unused mpq8785_of_match[] = { + { .compatible = "mps,mpq8785" }, + {} +}; +MODULE_DEVICE_TABLE(of, mpq8785_of_match); + +static struct i2c_driver mpq8785_driver = { + .driver = { + .name = "mpq8785", + .of_match_table = of_match_ptr(mpq8785_of_match), + }, + .probe = mpq8785_probe, + .id_table = mpq8785_id, +}; + +module_i2c_driver(mpq8785_driver); + +MODULE_AUTHOR("Charles Hsu "); +MODULE_DESCRIPTION("PMBus driver for MPS MPQ8785"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS(PMBUS); diff --git a/drivers/hwmon/pmbus/pmbus_core.c b/drivers/hwmon/pmbus/pmbus_core.c index 1363d9f89181..cb4c65a7f288 100644 --- a/drivers/hwmon/pmbus/pmbus_core.c +++ b/drivers/hwmon/pmbus/pmbus_core.c @@ -3188,7 +3188,7 @@ static int pmbus_regulator_notify(struct pmbus_data *data, int page, int event) static int pmbus_write_smbalert_mask(struct i2c_client *client, u8 page, u8 reg, u8 val) { - return pmbus_write_word_data(client, page, PMBUS_SMBALERT_MASK, reg | (val << 8)); + return _pmbus_write_word_data(client, page, PMBUS_SMBALERT_MASK, reg | (val << 8)); } static irqreturn_t pmbus_fault_handler(int irq, void *pdata) diff --git a/drivers/hwmon/pmbus/tda38640.c b/drivers/hwmon/pmbus/tda38640.c index 09cd114b1736..c31889a036f0 100644 --- a/drivers/hwmon/pmbus/tda38640.c +++ b/drivers/hwmon/pmbus/tda38640.c @@ -15,7 +15,7 @@ #include "pmbus.h" static const struct regulator_desc __maybe_unused tda38640_reg_desc[] = { - PMBUS_REGULATOR("vout", 0), + PMBUS_REGULATOR_ONE("vout"), }; struct tda38640_data { diff --git a/drivers/hwmon/powr1220.c b/drivers/hwmon/powr1220.c index 4120cadb00ae..2388d0565e7e 100644 --- a/drivers/hwmon/powr1220.c +++ b/drivers/hwmon/powr1220.c @@ -323,7 +323,6 @@ static const struct i2c_device_id powr1220_ids[] = { MODULE_DEVICE_TABLE(i2c, powr1220_ids); static struct i2c_driver powr1220_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "powr1220", }, diff --git a/drivers/hwmon/pt5161l.c b/drivers/hwmon/pt5161l.c new file mode 100644 index 000000000000..60361e39c474 --- /dev/null +++ b/drivers/hwmon/pt5161l.c @@ -0,0 +1,667 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include +#include +#include +#include +#include +#include +#include +#include + +/* Aries current average temp ADC code CSR */ +#define ARIES_CURRENT_AVG_TEMP_ADC_CSR 0x42c + +/* Device Load check register */ +#define ARIES_CODE_LOAD_REG 0x605 +/* Value indicating FW was loaded properly, [3:1] = 3'b111 */ +#define ARIES_LOAD_CODE 0xe + +/* Main Micro Heartbeat register */ +#define ARIES_MM_HEARTBEAT_ADDR 0x923 + +/* Reg offset to specify Address for MM assisted accesses */ +#define ARIES_MM_ASSIST_REG_ADDR_OFFSET 0xd99 +/* Reg offset to specify Command for MM assisted accesses */ +#define ARIES_MM_ASSIST_CMD_OFFSET 0xd9d +/* Reg offset to MM SPARE 0 used specify Address[7:0] */ +#define ARIES_MM_ASSIST_SPARE_0_OFFSET 0xd9f +/* Reg offset to MM SPARE 3 used specify Data Byte 0 */ +#define ARIES_MM_ASSIST_SPARE_3_OFFSET 0xda2 +/* Wide register reads */ +#define ARIES_MM_RD_WIDE_REG_2B 0x1d +#define ARIES_MM_RD_WIDE_REG_3B 0x1e +#define ARIES_MM_RD_WIDE_REG_4B 0x1f +#define ARIES_MM_RD_WIDE_REG_5B 0x20 + +/* Time delay between checking MM status of EEPROM write (microseconds) */ +#define ARIES_MM_STATUS_TIME 5000 + +/* AL Main SRAM DMEM offset (A0) */ +#define AL_MAIN_SRAM_DMEM_OFFSET (64 * 1024) +/* SRAM read command */ +#define AL_TG_RD_LOC_IND_SRAM 0x16 + +/* Offset for main micro FW info */ +#define ARIES_MAIN_MICRO_FW_INFO (96 * 1024 - 128) +/* FW Info (Major) offset location in struct */ +#define ARIES_MM_FW_VERSION_MAJOR 0 +/* FW Info (Minor) offset location in struct */ +#define ARIES_MM_FW_VERSION_MINOR 1 +/* FW Info (Build no.) offset location in struct */ +#define ARIES_MM_FW_VERSION_BUILD 2 + +#define ARIES_TEMP_CAL_CODE_DEFAULT 84 + +/* Struct defining FW version loaded on an Aries device */ +struct pt5161l_fw_ver { + u8 major; + u8 minor; + u16 build; +}; + +/* Each client has this additional data */ +struct pt5161l_data { + struct i2c_client *client; + struct dentry *debugfs; + struct pt5161l_fw_ver fw_ver; + struct mutex lock; /* for atomic I2C transactions */ + bool init_done; + bool code_load_okay; /* indicate if code load reg value is expected */ + bool mm_heartbeat_okay; /* indicate if Main Micro heartbeat is good */ + bool mm_wide_reg_access; /* MM assisted wide register access */ +}; + +static struct dentry *pt5161l_debugfs_dir; + +/* + * Write multiple data bytes to Aries over I2C + */ +static int pt5161l_write_block_data(struct pt5161l_data *data, u32 address, + u8 len, u8 *val) +{ + struct i2c_client *client = data->client; + int ret; + u8 remain_len = len; + u8 xfer_len, curr_len; + u8 buf[16]; + u8 cmd = 0x0F; /* [7]:pec_en, [4:2]:func, [1]:start, [0]:end */ + u8 config = 0x40; /* [6]:cfg_type, [4:1]:burst_len, [0]:address bit16 */ + + while (remain_len > 0) { + if (remain_len > 4) { + curr_len = 4; + remain_len -= 4; + } else { + curr_len = remain_len; + remain_len = 0; + } + + buf[0] = config | (curr_len - 1) << 1 | ((address >> 16) & 0x1); + buf[1] = (address >> 8) & 0xff; + buf[2] = address & 0xff; + memcpy(&buf[3], val, curr_len); + + xfer_len = 3 + curr_len; + ret = i2c_smbus_write_block_data(client, cmd, xfer_len, buf); + if (ret) + return ret; + + val += curr_len; + address += curr_len; + } + + return 0; +} + +/* + * Read multiple data bytes from Aries over I2C + */ +static int pt5161l_read_block_data(struct pt5161l_data *data, u32 address, + u8 len, u8 *val) +{ + struct i2c_client *client = data->client; + int ret, tries; + u8 remain_len = len; + u8 curr_len; + u8 wbuf[16], rbuf[24]; + u8 cmd = 0x08; /* [7]:pec_en, [4:2]:func, [1]:start, [0]:end */ + u8 config = 0x00; /* [6]:cfg_type, [4:1]:burst_len, [0]:address bit16 */ + + while (remain_len > 0) { + if (remain_len > 16) { + curr_len = 16; + remain_len -= 16; + } else { + curr_len = remain_len; + remain_len = 0; + } + + wbuf[0] = config | (curr_len - 1) << 1 | + ((address >> 16) & 0x1); + wbuf[1] = (address >> 8) & 0xff; + wbuf[2] = address & 0xff; + + for (tries = 0; tries < 3; tries++) { + ret = i2c_smbus_write_block_data(client, (cmd | 0x2), 3, + wbuf); + if (ret) + return ret; + + ret = i2c_smbus_read_block_data(client, (cmd | 0x1), + rbuf); + if (ret == curr_len) + break; + } + if (tries >= 3) + return ret; + + memcpy(val, rbuf, curr_len); + val += curr_len; + address += curr_len; + } + + return 0; +} + +static int pt5161l_read_wide_reg(struct pt5161l_data *data, u32 address, + u8 width, u8 *val) +{ + int ret, tries; + u8 buf[8]; + u8 status; + + /* + * Safely access wide registers using mailbox method to prevent + * risking conflict with Aries firmware; otherwise fallback to + * legacy, less secure method. + */ + if (data->mm_wide_reg_access) { + buf[0] = address & 0xff; + buf[1] = (address >> 8) & 0xff; + buf[2] = (address >> 16) & 0x1; + ret = pt5161l_write_block_data(data, + ARIES_MM_ASSIST_SPARE_0_OFFSET, + 3, buf); + if (ret) + return ret; + + /* Set command based on width */ + switch (width) { + case 2: + buf[0] = ARIES_MM_RD_WIDE_REG_2B; + break; + case 3: + buf[0] = ARIES_MM_RD_WIDE_REG_3B; + break; + case 4: + buf[0] = ARIES_MM_RD_WIDE_REG_4B; + break; + case 5: + buf[0] = ARIES_MM_RD_WIDE_REG_5B; + break; + default: + return -EINVAL; + } + ret = pt5161l_write_block_data(data, ARIES_MM_ASSIST_CMD_OFFSET, + 1, buf); + if (ret) + return ret; + + status = 0xff; + for (tries = 0; tries < 100; tries++) { + ret = pt5161l_read_block_data(data, + ARIES_MM_ASSIST_CMD_OFFSET, + 1, &status); + if (ret) + return ret; + + if (status == 0) + break; + + usleep_range(ARIES_MM_STATUS_TIME, + ARIES_MM_STATUS_TIME + 1000); + } + if (status != 0) + return -ETIMEDOUT; + + ret = pt5161l_read_block_data(data, + ARIES_MM_ASSIST_SPARE_3_OFFSET, + width, val); + if (ret) + return ret; + } else { + return pt5161l_read_block_data(data, address, width, val); + } + + return 0; +} + +/* + * Read multiple (up to eight) data bytes from micro SRAM over I2C + */ +static int +pt5161l_read_block_data_main_micro_indirect(struct pt5161l_data *data, + u32 address, u8 len, u8 *val) +{ + int ret, tries; + u8 buf[8]; + u8 i, status; + u32 uind_offs = ARIES_MM_ASSIST_REG_ADDR_OFFSET; + u32 eeprom_base, eeprom_addr; + + /* No multi-byte indirect support here. Hence read a byte at a time */ + eeprom_base = address - AL_MAIN_SRAM_DMEM_OFFSET; + for (i = 0; i < len; i++) { + eeprom_addr = eeprom_base + i; + buf[0] = eeprom_addr & 0xff; + buf[1] = (eeprom_addr >> 8) & 0xff; + buf[2] = (eeprom_addr >> 16) & 0xff; + ret = pt5161l_write_block_data(data, uind_offs, 3, buf); + if (ret) + return ret; + + buf[0] = AL_TG_RD_LOC_IND_SRAM; + ret = pt5161l_write_block_data(data, uind_offs + 4, 1, buf); + if (ret) + return ret; + + status = 0xff; + for (tries = 0; tries < 255; tries++) { + ret = pt5161l_read_block_data(data, uind_offs + 4, 1, + &status); + if (ret) + return ret; + + if (status == 0) + break; + } + if (status != 0) + return -ETIMEDOUT; + + ret = pt5161l_read_block_data(data, uind_offs + 3, 1, buf); + if (ret) + return ret; + + val[i] = buf[0]; + } + + return 0; +} + +/* + * Check firmware load status + */ +static int pt5161l_fw_load_check(struct pt5161l_data *data) +{ + int ret; + u8 buf[8]; + + ret = pt5161l_read_block_data(data, ARIES_CODE_LOAD_REG, 1, buf); + if (ret) + return ret; + + if (buf[0] < ARIES_LOAD_CODE) { + dev_dbg(&data->client->dev, + "Code Load reg unexpected. Not all modules are loaded %x\n", + buf[0]); + data->code_load_okay = false; + } else { + data->code_load_okay = true; + } + + return 0; +} + +/* + * Check main micro heartbeat + */ +static int pt5161l_heartbeat_check(struct pt5161l_data *data) +{ + int ret, tries; + u8 buf[8]; + u8 heartbeat; + bool hb_changed = false; + + ret = pt5161l_read_block_data(data, ARIES_MM_HEARTBEAT_ADDR, 1, buf); + if (ret) + return ret; + + heartbeat = buf[0]; + for (tries = 0; tries < 100; tries++) { + ret = pt5161l_read_block_data(data, ARIES_MM_HEARTBEAT_ADDR, 1, + buf); + if (ret) + return ret; + + if (buf[0] != heartbeat) { + hb_changed = true; + break; + } + } + data->mm_heartbeat_okay = hb_changed; + + return 0; +} + +/* + * Check the status of firmware + */ +static int pt5161l_fwsts_check(struct pt5161l_data *data) +{ + int ret; + u8 buf[8]; + u8 major = 0, minor = 0; + u16 build = 0; + + ret = pt5161l_fw_load_check(data); + if (ret) + return ret; + + ret = pt5161l_heartbeat_check(data); + if (ret) + return ret; + + if (data->code_load_okay && data->mm_heartbeat_okay) { + ret = pt5161l_read_block_data_main_micro_indirect(data, ARIES_MAIN_MICRO_FW_INFO + + ARIES_MM_FW_VERSION_MAJOR, + 1, &major); + if (ret) + return ret; + + ret = pt5161l_read_block_data_main_micro_indirect(data, ARIES_MAIN_MICRO_FW_INFO + + ARIES_MM_FW_VERSION_MINOR, + 1, &minor); + if (ret) + return ret; + + ret = pt5161l_read_block_data_main_micro_indirect(data, ARIES_MAIN_MICRO_FW_INFO + + ARIES_MM_FW_VERSION_BUILD, + 2, buf); + if (ret) + return ret; + build = buf[1] << 8 | buf[0]; + } + data->fw_ver.major = major; + data->fw_ver.minor = minor; + data->fw_ver.build = build; + + return 0; +} + +static int pt5161l_fw_is_at_least(struct pt5161l_data *data, u8 major, u8 minor, + u16 build) +{ + u32 ver = major << 24 | minor << 16 | build; + u32 curr_ver = data->fw_ver.major << 24 | data->fw_ver.minor << 16 | + data->fw_ver.build; + + if (curr_ver >= ver) + return true; + + return false; +} + +static int pt5161l_init_dev(struct pt5161l_data *data) +{ + int ret; + + mutex_lock(&data->lock); + ret = pt5161l_fwsts_check(data); + mutex_unlock(&data->lock); + if (ret) + return ret; + + /* Firmware 2.2.0 enables safe access to wide registers */ + if (pt5161l_fw_is_at_least(data, 2, 2, 0)) + data->mm_wide_reg_access = true; + + data->init_done = true; + + return 0; +} + +static int pt5161l_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *val) +{ + struct pt5161l_data *data = dev_get_drvdata(dev); + int ret; + u8 buf[8]; + long adc_code; + + switch (attr) { + case hwmon_temp_input: + if (!data->init_done) { + ret = pt5161l_init_dev(data); + if (ret) + return ret; + } + + mutex_lock(&data->lock); + ret = pt5161l_read_wide_reg(data, + ARIES_CURRENT_AVG_TEMP_ADC_CSR, 4, + buf); + mutex_unlock(&data->lock); + if (ret) { + dev_dbg(dev, "Read adc_code failed %d\n", ret); + return ret; + } + + adc_code = buf[3] << 24 | buf[2] << 16 | buf[1] << 8 | buf[0]; + if (adc_code == 0 || adc_code >= 0x3ff) { + dev_dbg(dev, "Invalid adc_code %lx\n", adc_code); + return -EIO; + } + + *val = 110000 + + ((adc_code - (ARIES_TEMP_CAL_CODE_DEFAULT + 250)) * + -320); + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + +static umode_t pt5161l_is_visible(const void *data, + enum hwmon_sensor_types type, u32 attr, + int channel) +{ + switch (attr) { + case hwmon_temp_input: + return 0444; + default: + break; + } + + return 0; +} + +static const struct hwmon_channel_info *pt5161l_info[] = { + HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT), + NULL +}; + +static const struct hwmon_ops pt5161l_hwmon_ops = { + .is_visible = pt5161l_is_visible, + .read = pt5161l_read, +}; + +static const struct hwmon_chip_info pt5161l_chip_info = { + .ops = &pt5161l_hwmon_ops, + .info = pt5161l_info, +}; + +static ssize_t pt5161l_debugfs_read_fw_ver(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct pt5161l_data *data = file->private_data; + int ret; + char ver[32]; + + mutex_lock(&data->lock); + ret = pt5161l_fwsts_check(data); + mutex_unlock(&data->lock); + if (ret) + return ret; + + ret = snprintf(ver, sizeof(ver), "%u.%u.%u\n", data->fw_ver.major, + data->fw_ver.minor, data->fw_ver.build); + + return simple_read_from_buffer(buf, count, ppos, ver, ret); +} + +static const struct file_operations pt5161l_debugfs_ops_fw_ver = { + .read = pt5161l_debugfs_read_fw_ver, + .open = simple_open, +}; + +static ssize_t pt5161l_debugfs_read_fw_load_sts(struct file *file, + char __user *buf, size_t count, + loff_t *ppos) +{ + struct pt5161l_data *data = file->private_data; + int ret; + bool status = false; + char health[16]; + + mutex_lock(&data->lock); + ret = pt5161l_fw_load_check(data); + mutex_unlock(&data->lock); + if (ret == 0) + status = data->code_load_okay; + + ret = snprintf(health, sizeof(health), "%s\n", + status ? "normal" : "abnormal"); + + return simple_read_from_buffer(buf, count, ppos, health, ret); +} + +static const struct file_operations pt5161l_debugfs_ops_fw_load_sts = { + .read = pt5161l_debugfs_read_fw_load_sts, + .open = simple_open, +}; + +static ssize_t pt5161l_debugfs_read_hb_sts(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct pt5161l_data *data = file->private_data; + int ret; + bool status = false; + char health[16]; + + mutex_lock(&data->lock); + ret = pt5161l_heartbeat_check(data); + mutex_unlock(&data->lock); + if (ret == 0) + status = data->mm_heartbeat_okay; + + ret = snprintf(health, sizeof(health), "%s\n", + status ? "normal" : "abnormal"); + + return simple_read_from_buffer(buf, count, ppos, health, ret); +} + +static const struct file_operations pt5161l_debugfs_ops_hb_sts = { + .read = pt5161l_debugfs_read_hb_sts, + .open = simple_open, +}; + +static int pt5161l_init_debugfs(struct pt5161l_data *data) +{ + data->debugfs = debugfs_create_dir(dev_name(&data->client->dev), + pt5161l_debugfs_dir); + + debugfs_create_file("fw_ver", 0444, data->debugfs, data, + &pt5161l_debugfs_ops_fw_ver); + + debugfs_create_file("fw_load_status", 0444, data->debugfs, data, + &pt5161l_debugfs_ops_fw_load_sts); + + debugfs_create_file("heartbeat_status", 0444, data->debugfs, data, + &pt5161l_debugfs_ops_hb_sts); + + return 0; +} + +static int pt5161l_probe(struct i2c_client *client) +{ + struct device *dev = &client->dev; + struct device *hwmon_dev; + struct pt5161l_data *data; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->client = client; + mutex_init(&data->lock); + pt5161l_init_dev(data); + dev_set_drvdata(dev, data); + + hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name, + data, + &pt5161l_chip_info, + NULL); + + pt5161l_init_debugfs(data); + + return PTR_ERR_OR_ZERO(hwmon_dev); +} + +static void pt5161l_remove(struct i2c_client *client) +{ + struct pt5161l_data *data = i2c_get_clientdata(client); + + debugfs_remove_recursive(data->debugfs); +} + +static const struct of_device_id __maybe_unused pt5161l_of_match[] = { + { .compatible = "asteralabs,pt5161l" }, + {}, +}; +MODULE_DEVICE_TABLE(of, pt5161l_of_match); + +static const struct acpi_device_id __maybe_unused pt5161l_acpi_match[] = { + { "PT5161L", 0 }, + {}, +}; +MODULE_DEVICE_TABLE(acpi, pt5161l_acpi_match); + +static const struct i2c_device_id pt5161l_id[] = { + { "pt5161l", 0 }, + {} +}; +MODULE_DEVICE_TABLE(i2c, pt5161l_id); + +static struct i2c_driver pt5161l_driver = { + .class = I2C_CLASS_HWMON, + .driver = { + .name = "pt5161l", + .of_match_table = of_match_ptr(pt5161l_of_match), + .acpi_match_table = ACPI_PTR(pt5161l_acpi_match), + }, + .probe = pt5161l_probe, + .remove = pt5161l_remove, + .id_table = pt5161l_id, +}; + +static int __init pt5161l_init(void) +{ + pt5161l_debugfs_dir = debugfs_create_dir("pt5161l", NULL); + return i2c_add_driver(&pt5161l_driver); +} + +static void __exit pt5161l_exit(void) +{ + i2c_del_driver(&pt5161l_driver); + debugfs_remove_recursive(pt5161l_debugfs_dir); +} + +module_init(pt5161l_init); +module_exit(pt5161l_exit); + +MODULE_AUTHOR("Cosmo Chou "); +MODULE_DESCRIPTION("Hwmon driver for Astera Labs Aries PCIe retimer"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hwmon/sbrmi.c b/drivers/hwmon/sbrmi.c index 484703f0ea5f..4318f5121145 100644 --- a/drivers/hwmon/sbrmi.c +++ b/drivers/hwmon/sbrmi.c @@ -342,7 +342,6 @@ static const struct of_device_id __maybe_unused sbrmi_of_match[] = { MODULE_DEVICE_TABLE(of, sbrmi_of_match); static struct i2c_driver sbrmi_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "sbrmi", .of_match_table = of_match_ptr(sbrmi_of_match), diff --git a/drivers/hwmon/sbtsi_temp.c b/drivers/hwmon/sbtsi_temp.c index dd85cf89f008..a4181acb1aa6 100644 --- a/drivers/hwmon/sbtsi_temp.c +++ b/drivers/hwmon/sbtsi_temp.c @@ -232,7 +232,6 @@ static const struct of_device_id __maybe_unused sbtsi_of_match[] = { MODULE_DEVICE_TABLE(of, sbtsi_of_match); static struct i2c_driver sbtsi_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "sbtsi", .of_match_table = of_match_ptr(sbtsi_of_match), diff --git a/drivers/hwmon/sch5627.c b/drivers/hwmon/sch5627.c index 1891d4d75aa9..33e997b5c1f5 100644 --- a/drivers/hwmon/sch5627.c +++ b/drivers/hwmon/sch5627.c @@ -116,7 +116,7 @@ static const struct regmap_config sch5627_regmap_config = { .val_bits = 8, .wr_table = &sch5627_tunables_table, .rd_table = &sch5627_tunables_table, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, .use_single_read = true, .use_single_write = true, .can_sleep = true, diff --git a/drivers/hwmon/sht3x.c b/drivers/hwmon/sht3x.c index 79657910b79e..c0d02fbcdb76 100644 --- a/drivers/hwmon/sht3x.c +++ b/drivers/hwmon/sht3x.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -41,6 +42,9 @@ static const unsigned char sht3x_cmd_heater_off[] = { 0x30, 0x66 }; /* other commands */ static const unsigned char sht3x_cmd_read_status_reg[] = { 0xf3, 0x2d }; static const unsigned char sht3x_cmd_clear_status_reg[] = { 0x30, 0x41 }; +static const unsigned char sht3x_cmd_read_serial_number[] = { 0x37, 0x80 }; + +static struct dentry *debugfs; /* delays for single-shot mode i2c commands, both in us */ #define SHT3X_SINGLE_WAIT_TIME_HPM 15000 @@ -163,12 +167,14 @@ struct sht3x_data { enum sht3x_chips chip_id; struct mutex i2c_lock; /* lock for sending i2c commands */ struct mutex data_lock; /* lock for updating driver data */ + struct dentry *sensor_dir; u8 mode; const unsigned char *command; u32 wait_time; /* in us*/ unsigned long last_update; /* last update in periodic mode*/ enum sht3x_repeatability repeatability; + u32 serial_number; /* * cached values for temperature and humidity and limits @@ -831,6 +837,40 @@ static int sht3x_write(struct device *dev, enum hwmon_sensor_types type, } } +static void sht3x_debugfs_init(struct sht3x_data *data) +{ + char name[32]; + + snprintf(name, sizeof(name), "i2c%u-%02x", + data->client->adapter->nr, data->client->addr); + data->sensor_dir = debugfs_create_dir(name, debugfs); + debugfs_create_u32("serial_number", 0444, + data->sensor_dir, &data->serial_number); +} + +static void sht3x_debugfs_remove(void *sensor_dir) +{ + debugfs_remove_recursive(sensor_dir); +} + +static int sht3x_serial_number_read(struct sht3x_data *data) +{ + int ret; + char buffer[SHT3X_RESPONSE_LENGTH]; + struct i2c_client *client = data->client; + + ret = sht3x_read_from_command(client, data, + sht3x_cmd_read_serial_number, + buffer, + SHT3X_RESPONSE_LENGTH, 0); + if (ret) + return ret; + + data->serial_number = (buffer[0] << 24) | (buffer[1] << 16) | + (buffer[3] << 8) | buffer[4]; + return ret; +} + static const struct hwmon_ops sht3x_ops = { .is_visible = sht3x_is_visible, .read = sht3x_read, @@ -899,6 +939,18 @@ static int sht3x_probe(struct i2c_client *client) if (ret) return ret; + ret = sht3x_serial_number_read(data); + if (ret) { + dev_dbg(dev, "unable to read serial number\n"); + } else { + sht3x_debugfs_init(data); + ret = devm_add_action_or_reset(dev, + sht3x_debugfs_remove, + data->sensor_dir); + if (ret) + return ret; + } + hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name, data, @@ -917,7 +969,19 @@ static struct i2c_driver sht3x_i2c_driver = { .id_table = sht3x_ids, }; -module_i2c_driver(sht3x_i2c_driver); +static int __init sht3x_init(void) +{ + debugfs = debugfs_create_dir("sht3x", NULL); + return i2c_add_driver(&sht3x_i2c_driver); +} +module_init(sht3x_init); + +static void __exit sht3x_cleanup(void) +{ + debugfs_remove_recursive(debugfs); + i2c_del_driver(&sht3x_i2c_driver); +} +module_exit(sht3x_cleanup); MODULE_AUTHOR("David Frey "); MODULE_AUTHOR("Pascal Sachs "); diff --git a/drivers/hwmon/sis5595.c b/drivers/hwmon/sis5595.c index 641be1f7f9cd..e73b1522f3ce 100644 --- a/drivers/hwmon/sis5595.c +++ b/drivers/hwmon/sis5595.c @@ -153,13 +153,9 @@ static inline s8 TEMP_TO_REG(long val) } /* - * FAN DIV: 1, 2, 4, or 8 (defaults to 2) - * REG: 0, 1, 2, or 3 (respectively) (defaults to 1) + * FAN DIV: 1, 2, 4, or 8 + * REG: 0, 1, 2, or 3 (respectively) */ -static inline u8 DIV_TO_REG(int val) -{ - return val == 8 ? 3 : val == 4 ? 2 : val == 1 ? 0 : 1; -} #define DIV_FROM_REG(val) (1 << (val)) /* diff --git a/drivers/hwmon/surface_fan.c b/drivers/hwmon/surface_fan.c new file mode 100644 index 000000000000..de3c5a2409c6 --- /dev/null +++ b/drivers/hwmon/surface_fan.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Surface Fan driver for Surface System Aggregator Module. It provides access + * to the fan's rpm through the hwmon system. + * + * Copyright (C) 2023 Ivor Wanders + */ + +#include +#include +#include +#include +#include + +// SSAM +SSAM_DEFINE_SYNC_REQUEST_CL_R(__ssam_fan_rpm_get, __le16, { + .target_category = SSAM_SSH_TC_FAN, + .command_id = 0x01, +}); + +// hwmon +static umode_t surface_fan_hwmon_is_visible(const void *drvdata, + enum hwmon_sensor_types type, u32 attr, + int channel) +{ + return 0444; +} + +static int surface_fan_hwmon_read(struct device *dev, + enum hwmon_sensor_types type, u32 attr, + int channel, long *val) +{ + struct ssam_device *sdev = dev_get_drvdata(dev); + int ret; + __le16 value; + + ret = __ssam_fan_rpm_get(sdev, &value); + if (ret) + return ret; + + *val = le16_to_cpu(value); + + return 0; +} + +static const struct hwmon_channel_info *const surface_fan_info[] = { + HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT), + NULL +}; + +static const struct hwmon_ops surface_fan_hwmon_ops = { + .is_visible = surface_fan_hwmon_is_visible, + .read = surface_fan_hwmon_read, +}; + +static const struct hwmon_chip_info surface_fan_chip_info = { + .ops = &surface_fan_hwmon_ops, + .info = surface_fan_info, +}; + +static int surface_fan_probe(struct ssam_device *sdev) +{ + struct device *hdev; + + hdev = devm_hwmon_device_register_with_info(&sdev->dev, + "surface_fan", sdev, + &surface_fan_chip_info, + NULL); + + return PTR_ERR_OR_ZERO(hdev); +} + +static const struct ssam_device_id ssam_fan_match[] = { + { SSAM_SDEV(FAN, SAM, 0x01, 0x01) }, + {}, +}; +MODULE_DEVICE_TABLE(ssam, ssam_fan_match); + +static struct ssam_device_driver surface_fan = { + .probe = surface_fan_probe, + .match_table = ssam_fan_match, + .driver = { + .name = "surface_fan", + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, +}; +module_ssam_device_driver(surface_fan); + +MODULE_AUTHOR("Ivor Wanders "); +MODULE_DESCRIPTION("Fan Driver for Surface System Aggregator Module"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hwmon/tmp401.c b/drivers/hwmon/tmp401.c index 91f2314568cf..df1b45a62e80 100644 --- a/drivers/hwmon/tmp401.c +++ b/drivers/hwmon/tmp401.c @@ -256,7 +256,7 @@ static int tmp401_reg_write(void *context, unsigned int reg, unsigned int val) static const struct regmap_config tmp401_regmap_config = { .reg_bits = 8, .val_bits = 16, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, .volatile_reg = tmp401_regmap_is_volatile, .reg_read = tmp401_reg_read, .reg_write = tmp401_reg_write, diff --git a/drivers/hwmon/w83773g.c b/drivers/hwmon/w83773g.c index 045eea8378c2..401a28f55f93 100644 --- a/drivers/hwmon/w83773g.c +++ b/drivers/hwmon/w83773g.c @@ -290,7 +290,6 @@ static int w83773_probe(struct i2c_client *client) } static struct i2c_driver w83773_driver = { - .class = I2C_CLASS_HWMON, .driver = { .name = "w83773g", .of_match_table = of_match_ptr(w83773_of_match), diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index bcf1198e8991..e486027f8b07 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -1934,7 +1934,8 @@ static void __init spr_idle_state_table_update(void) static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) { - unsigned int mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint) + 1; + unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) & + MWAIT_CSTATE_MASK; unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) & MWAIT_SUBSTATE_MASK; diff --git a/drivers/iio/adc/ad_sigma_delta.c b/drivers/iio/adc/ad_sigma_delta.c index 7e2192870743..55442eddf57c 100644 --- a/drivers/iio/adc/ad_sigma_delta.c +++ b/drivers/iio/adc/ad_sigma_delta.c @@ -212,7 +212,7 @@ int ad_sd_calibrate(struct ad_sigma_delta *sigma_delta, if (ret) return ret; - spi_bus_lock(sigma_delta->spi->master); + spi_bus_lock(sigma_delta->spi->controller); sigma_delta->bus_locked = true; sigma_delta->keep_cs_asserted = true; reinit_completion(&sigma_delta->completion); @@ -235,7 +235,7 @@ out: sigma_delta->keep_cs_asserted = false; ad_sigma_delta_set_mode(sigma_delta, AD_SD_MODE_IDLE); sigma_delta->bus_locked = false; - spi_bus_unlock(sigma_delta->spi->master); + spi_bus_unlock(sigma_delta->spi->controller); return ret; } @@ -287,7 +287,7 @@ int ad_sigma_delta_single_conversion(struct iio_dev *indio_dev, ad_sigma_delta_set_channel(sigma_delta, chan->address); - spi_bus_lock(sigma_delta->spi->master); + spi_bus_lock(sigma_delta->spi->controller); sigma_delta->bus_locked = true; sigma_delta->keep_cs_asserted = true; reinit_completion(&sigma_delta->completion); @@ -322,7 +322,7 @@ out: sigma_delta->keep_cs_asserted = false; ad_sigma_delta_set_mode(sigma_delta, AD_SD_MODE_IDLE); sigma_delta->bus_locked = false; - spi_bus_unlock(sigma_delta->spi->master); + spi_bus_unlock(sigma_delta->spi->controller); iio_device_release_direct_mode(indio_dev); if (ret) @@ -387,7 +387,7 @@ static int ad_sd_buffer_postenable(struct iio_dev *indio_dev) sigma_delta->samples_buf = samples_buf; - spi_bus_lock(sigma_delta->spi->master); + spi_bus_lock(sigma_delta->spi->controller); sigma_delta->bus_locked = true; sigma_delta->keep_cs_asserted = true; @@ -401,7 +401,7 @@ static int ad_sd_buffer_postenable(struct iio_dev *indio_dev) return 0; err_unlock: - spi_bus_unlock(sigma_delta->spi->master); + spi_bus_unlock(sigma_delta->spi->controller); return ret; } @@ -426,7 +426,7 @@ static int ad_sd_buffer_postdisable(struct iio_dev *indio_dev) ad_sigma_delta_disable_all(sigma_delta); sigma_delta->bus_locked = false; - return spi_bus_unlock(sigma_delta->spi->master); + return spi_bus_unlock(sigma_delta->spi->controller); } static irqreturn_t ad_sd_trigger_handler(int irq, void *p) diff --git a/drivers/input/joystick/psxpad-spi.c b/drivers/input/joystick/psxpad-spi.c index de734a927b4d..c47fc5f34bd0 100644 --- a/drivers/input/joystick/psxpad-spi.c +++ b/drivers/input/joystick/psxpad-spi.c @@ -342,8 +342,8 @@ static int psxpad_spi_probe(struct spi_device *spi) spi->mode = SPI_MODE_3; spi->bits_per_word = 8; /* (PlayStation 1/2 joypad might be possible works 250kHz/500kHz) */ - spi->master->min_speed_hz = 125000; - spi->master->max_speed_hz = 125000; + spi->controller->min_speed_hz = 125000; + spi->controller->max_speed_hz = 125000; spi_setup(spi); /* pad settings */ diff --git a/drivers/input/rmi4/rmi_spi.c b/drivers/input/rmi4/rmi_spi.c index 07c866f42296..9d92129aa432 100644 --- a/drivers/input/rmi4/rmi_spi.c +++ b/drivers/input/rmi4/rmi_spi.c @@ -375,7 +375,7 @@ static int rmi_spi_probe(struct spi_device *spi) struct rmi_device_platform_data *spi_pdata = spi->dev.platform_data; int error; - if (spi->master->flags & SPI_CONTROLLER_HALF_DUPLEX) + if (spi->controller->flags & SPI_CONTROLLER_HALF_DUPLEX) return -EINVAL; rmi_spi = devm_kzalloc(&spi->dev, sizeof(struct rmi_spi_xport), diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 0d6095290b6a..0af39bbbe3a3 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -163,6 +163,9 @@ config IOMMU_SVA select IOMMU_MM_DATA bool +config IOMMU_IOPF + bool + config FSL_PAMU bool "Freescale IOMMU support" depends on PCI @@ -196,7 +199,7 @@ source "drivers/iommu/iommufd/Kconfig" config IRQ_REMAP bool "Support for Interrupt Remapping" depends on X86_64 && X86_IO_APIC && PCI_MSI && ACPI - select DMAR_TABLE + select DMAR_TABLE if INTEL_IOMMU help Supports Interrupt remapping for IO-APIC and MSI devices. To use x2apic mode in the CPU's which support x2APIC enhancements or @@ -398,6 +401,7 @@ config ARM_SMMU_V3_SVA bool "Shared Virtual Addressing support for the ARM SMMUv3" depends on ARM_SMMU_V3 select IOMMU_SVA + select IOMMU_IOPF select MMU_NOTIFIER help Support for sharing process address spaces with devices using the diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 95ad9dbfbda0..542760d963ec 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -26,6 +26,7 @@ obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o obj-$(CONFIG_S390_IOMMU) += s390-iommu.o obj-$(CONFIG_HYPERV_IOMMU) += hyperv-iommu.o obj-$(CONFIG_VIRTIO_IOMMU) += virtio-iommu.o -obj-$(CONFIG_IOMMU_SVA) += iommu-sva.o io-pgfault.o +obj-$(CONFIG_IOMMU_SVA) += iommu-sva.o +obj-$(CONFIG_IOMMU_IOPF) += io-pgfault.o obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o obj-$(CONFIG_APPLE_DART) += apple-dart.o diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index c970eae2313d..f482aab420f7 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -39,20 +39,16 @@ extern enum io_pgtable_fmt amd_iommu_pgtable; extern int amd_iommu_gpt_level; bool amd_iommu_v2_supported(void); -struct amd_iommu *get_amd_iommu(unsigned int idx); -u8 amd_iommu_pc_get_max_banks(unsigned int idx); -bool amd_iommu_pc_supported(void); -u8 amd_iommu_pc_get_max_counters(unsigned int idx); -int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, - u8 fxn, u64 *value); -int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, - u8 fxn, u64 *value); /* Device capabilities */ int amd_iommu_pdev_enable_cap_pri(struct pci_dev *pdev); void amd_iommu_pdev_disable_cap_pri(struct pci_dev *pdev); -int amd_iommu_flush_page(struct iommu_domain *dom, u32 pasid, u64 address); +/* GCR3 setup */ +int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, + ioasid_t pasid, unsigned long gcr3); +int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid); + /* * This function flushes all internal caches of * the IOMMU used by this driver. @@ -63,10 +59,10 @@ void amd_iommu_domain_update(struct protection_domain *domain); void amd_iommu_domain_flush_complete(struct protection_domain *domain); void amd_iommu_domain_flush_pages(struct protection_domain *domain, u64 address, size_t size); -int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid); -int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, u32 pasid, - unsigned long cr3); -int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, u32 pasid); +void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data, + ioasid_t pasid, u64 address, size_t size); +void amd_iommu_dev_flush_pasid_all(struct iommu_dev_data *dev_data, + ioasid_t pasid); #ifdef CONFIG_IRQ_REMAP int amd_iommu_create_irq_domain(struct amd_iommu *iommu); @@ -77,10 +73,6 @@ static inline int amd_iommu_create_irq_domain(struct amd_iommu *iommu) } #endif -#define PPR_SUCCESS 0x0 -#define PPR_INVALID 0x1 -#define PPR_FAILURE 0xf - int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid, int status, int tag); @@ -150,6 +142,21 @@ static inline void *alloc_pgtable_page(int nid, gfp_t gfp) return page ? page_address(page) : NULL; } +/* + * This must be called after device probe completes. During probe + * use rlookup_amd_iommu() get the iommu. + */ +static inline struct amd_iommu *get_amd_iommu_from_dev(struct device *dev) +{ + return iommu_get_iommu_dev(dev, struct amd_iommu, iommu); +} + +/* This must be called after device probe completes. */ +static inline struct amd_iommu *get_amd_iommu_from_dev_data(struct iommu_dev_data *dev_data) +{ + return iommu_get_iommu_dev(dev_data->dev, struct amd_iommu, iommu); +} + bool translation_pre_enabled(struct amd_iommu *iommu); bool amd_iommu_is_attach_deferred(struct device *dev); int __init add_special_device(u8 type, u8 id, u32 *devid, bool cmd_line); diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 809d74faa1a5..d1fed5fc219b 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -453,15 +453,6 @@ #define MAX_DOMAIN_ID 65536 -/* Protection domain flags */ -#define PD_DMA_OPS_MASK BIT(0) /* domain used for dma_ops */ -#define PD_DEFAULT_MASK BIT(1) /* domain is a default dma_ops - domain for an IOMMU */ -#define PD_PASSTHROUGH_MASK BIT(2) /* domain has no page - translation */ -#define PD_IOMMUV2_MASK BIT(3) /* domain has gcr3 table */ -#define PD_GIOV_MASK BIT(4) /* domain enable GIOV support */ - /* Timeout stuff */ #define LOOP_TIMEOUT 100000 #define MMIO_STATUS_TIMEOUT 2000000 @@ -513,14 +504,6 @@ extern struct kmem_cache *amd_iommu_irq_cache; #define for_each_iommu_safe(iommu, next) \ list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list) -#define APERTURE_RANGE_SHIFT 27 /* 128 MB */ -#define APERTURE_RANGE_SIZE (1ULL << APERTURE_RANGE_SHIFT) -#define APERTURE_RANGE_PAGES (APERTURE_RANGE_SIZE >> PAGE_SHIFT) -#define APERTURE_MAX_RANGES 32 /* allows 4GB of DMA address space */ -#define APERTURE_RANGE_INDEX(a) ((a) >> APERTURE_RANGE_SHIFT) -#define APERTURE_PAGE_INDEX(a) (((a) >> 21) & 0x3fULL) - - struct amd_iommu; struct iommu_domain; struct irq_domain; @@ -541,6 +524,13 @@ struct amd_irte_ops; #define io_pgtable_cfg_to_data(x) \ container_of((x), struct amd_io_pgtable, pgtbl_cfg) +struct gcr3_tbl_info { + u64 *gcr3_tbl; /* Guest CR3 table */ + int glx; /* Number of levels for GCR3 table */ + u32 pasid_cnt; /* Track attached PASIDs */ + u16 domid; /* Per device domain ID */ +}; + struct amd_io_pgtable { struct io_pgtable_cfg pgtbl_cfg; struct io_pgtable iop; @@ -549,6 +539,11 @@ struct amd_io_pgtable { u64 *pgd; /* v2 pgtable pgd pointer */ }; +enum protection_domain_mode { + PD_MODE_V1 = 1, + PD_MODE_V2, +}; + /* * This structure contains generic data for IOMMU protection domains * independent of their use. @@ -560,10 +555,8 @@ struct protection_domain { struct amd_io_pgtable iop; spinlock_t lock; /* mostly used to lock the page table*/ u16 id; /* the domain id written to the device table */ - int glx; /* Number of levels for GCR3 table */ int nid; /* Node ID */ - u64 *gcr3_tbl; /* Guest CR3 table */ - unsigned long flags; /* flags to find out type of domain */ + enum protection_domain_mode pd_mode; /* Track page table type */ bool dirty_tracking; /* dirty tracking is enabled in the domain */ unsigned dev_cnt; /* devices assigned to this domain */ unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */ @@ -816,6 +809,7 @@ struct iommu_dev_data { struct list_head list; /* For domain->dev_list */ struct llist_node dev_data_list; /* For global dev_data_list */ struct protection_domain *domain; /* Domain the device is bound to */ + struct gcr3_tbl_info gcr3_info; /* Per-device GCR3 table */ struct device *dev; u16 devid; /* PCI Device ID */ diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 480e7681f4f3..e7a44929f0da 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -2069,6 +2069,9 @@ static int __init iommu_init_pci(struct amd_iommu *iommu) /* Prevent binding other PCI device drivers to IOMMU devices */ iommu->dev->match_driver = false; + /* ACPI _PRT won't have an IRQ for IOMMU */ + iommu->dev->irq_managed = 1; + pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET, &iommu->cap); @@ -2770,6 +2773,7 @@ static void early_enable_iommu(struct amd_iommu *iommu) iommu_enable_command_buffer(iommu); iommu_enable_event_buffer(iommu); iommu_set_exclusion_range(iommu); + iommu_enable_gt(iommu); iommu_enable_ga(iommu); iommu_enable_xt(iommu); iommu_enable_irtcachedis(iommu); @@ -2826,6 +2830,7 @@ static void early_enable_iommus(void) iommu_disable_irtcachedis(iommu); iommu_enable_command_buffer(iommu); iommu_enable_event_buffer(iommu); + iommu_enable_gt(iommu); iommu_enable_ga(iommu); iommu_enable_xt(iommu); iommu_enable_irtcachedis(iommu); @@ -2839,10 +2844,8 @@ static void enable_iommus_v2(void) { struct amd_iommu *iommu; - for_each_iommu(iommu) { + for_each_iommu(iommu) iommu_enable_ppr_log(iommu); - iommu_enable_gt(iommu); - } } static void enable_iommus_vapic(void) @@ -3726,13 +3729,11 @@ u8 amd_iommu_pc_get_max_banks(unsigned int idx) return 0; } -EXPORT_SYMBOL(amd_iommu_pc_get_max_banks); bool amd_iommu_pc_supported(void) { return amd_iommu_pc_present; } -EXPORT_SYMBOL(amd_iommu_pc_supported); u8 amd_iommu_pc_get_max_counters(unsigned int idx) { @@ -3743,7 +3744,6 @@ u8 amd_iommu_pc_get_max_counters(unsigned int idx) return 0; } -EXPORT_SYMBOL(amd_iommu_pc_get_max_counters); static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value, bool is_write) diff --git a/drivers/iommu/amd/io_pgtable_v2.c b/drivers/iommu/amd/io_pgtable_v2.c index 6d69ba60744f..93489d2db4e8 100644 --- a/drivers/iommu/amd/io_pgtable_v2.c +++ b/drivers/iommu/amd/io_pgtable_v2.c @@ -350,38 +350,26 @@ static const struct iommu_flush_ops v2_flush_ops = { static void v2_free_pgtable(struct io_pgtable *iop) { - struct protection_domain *pdom; struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, iop); - pdom = container_of(pgtable, struct protection_domain, iop); - if (!(pdom->flags & PD_IOMMUV2_MASK)) + if (!pgtable || !pgtable->pgd) return; - /* Clear gcr3 entry */ - amd_iommu_domain_clear_gcr3(&pdom->domain, 0); - - /* Make changes visible to IOMMUs */ - amd_iommu_domain_update(pdom); - /* Free page table */ free_pgtable(pgtable->pgd, get_pgtable_level()); + pgtable->pgd = NULL; } static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) { struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg); struct protection_domain *pdom = (struct protection_domain *)cookie; - int ret; int ias = IOMMU_IN_ADDR_BIT_SIZE; pgtable->pgd = alloc_pgtable_page(pdom->nid, GFP_ATOMIC); if (!pgtable->pgd) return NULL; - ret = amd_iommu_domain_set_gcr3(&pdom->domain, 0, iommu_virt_to_phys(pgtable->pgd)); - if (ret) - goto err_free_pgd; - if (get_pgtable_level() == PAGE_MODE_5_LEVEL) ias = 57; @@ -395,11 +383,6 @@ static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo cfg->tlb = &v2_flush_ops; return &pgtable->iop; - -err_free_pgd: - free_pgtable_page(pgtable->pgd); - - return NULL; } struct io_pgtable_init_fns io_pgtable_amd_iommu_v2_init_fns = { diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 4283dd8191f0..d35c1b8c8e65 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -45,10 +45,6 @@ #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) -/* IO virtual address start page frame number */ -#define IOVA_START_PFN (1) -#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) - /* Reserved IOVA ranges */ #define MSI_RANGE_START (0xfee00000) #define MSI_RANGE_END (0xfeefffff) @@ -79,6 +75,9 @@ struct kmem_cache *amd_iommu_irq_cache; static void detach_device(struct device *dev); +static void set_dte_entry(struct amd_iommu *iommu, + struct iommu_dev_data *dev_data); + /**************************************************************************** * * Helper functions @@ -87,7 +86,7 @@ static void detach_device(struct device *dev); static inline bool pdom_is_v2_pgtbl_mode(struct protection_domain *pdom) { - return (pdom && (pdom->flags & PD_IOMMUV2_MASK)); + return (pdom && (pdom->pd_mode == PD_MODE_V2)); } static inline int get_acpihid_device_id(struct device *dev, @@ -1388,14 +1387,9 @@ void amd_iommu_flush_all_caches(struct amd_iommu *iommu) static int device_flush_iotlb(struct iommu_dev_data *dev_data, u64 address, size_t size, ioasid_t pasid, bool gn) { - struct amd_iommu *iommu; + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); struct iommu_cmd cmd; - int qdep; - - qdep = dev_data->ats_qdep; - iommu = rlookup_amd_iommu(dev_data->dev); - if (!iommu) - return -EINVAL; + int qdep = dev_data->ats_qdep; build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size, pasid, gn); @@ -1415,16 +1409,12 @@ static int device_flush_dte_alias(struct pci_dev *pdev, u16 alias, void *data) */ static int device_flush_dte(struct iommu_dev_data *dev_data) { - struct amd_iommu *iommu; + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); struct pci_dev *pdev = NULL; struct amd_iommu_pci_seg *pci_seg; u16 alias; int ret; - iommu = rlookup_amd_iommu(dev_data->dev); - if (!iommu) - return -EINVAL; - if (dev_is_pci(dev_data->dev)) pdev = to_pci_dev(dev_data->dev); @@ -1453,27 +1443,37 @@ static int device_flush_dte(struct iommu_dev_data *dev_data) return ret; } -/* - * TLB invalidation function which is called from the mapping functions. - * It invalidates a single PTE if the range to flush is within a single - * page. Otherwise it flushes the whole TLB of the IOMMU. - */ -static void __domain_flush_pages(struct protection_domain *domain, +static int domain_flush_pages_v2(struct protection_domain *pdom, u64 address, size_t size) { struct iommu_dev_data *dev_data; + struct iommu_cmd cmd; + int ret = 0; + + list_for_each_entry(dev_data, &pdom->dev_list, list) { + struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); + u16 domid = dev_data->gcr3_info.domid; + + build_inv_iommu_pages(&cmd, address, size, + domid, IOMMU_NO_PASID, true); + + ret |= iommu_queue_command(iommu, &cmd); + } + + return ret; +} + +static int domain_flush_pages_v1(struct protection_domain *pdom, + u64 address, size_t size) +{ struct iommu_cmd cmd; int ret = 0, i; - ioasid_t pasid = IOMMU_NO_PASID; - bool gn = false; - if (pdom_is_v2_pgtbl_mode(domain)) - gn = true; - - build_inv_iommu_pages(&cmd, address, size, domain->id, pasid, gn); + build_inv_iommu_pages(&cmd, address, size, + pdom->id, IOMMU_NO_PASID, false); for (i = 0; i < amd_iommu_get_num_iommus(); ++i) { - if (!domain->dev_iommu[i]) + if (!pdom->dev_iommu[i]) continue; /* @@ -1483,6 +1483,28 @@ static void __domain_flush_pages(struct protection_domain *domain, ret |= iommu_queue_command(amd_iommus[i], &cmd); } + return ret; +} + +/* + * TLB invalidation function which is called from the mapping functions. + * It flushes range of PTEs of the domain. + */ +static void __domain_flush_pages(struct protection_domain *domain, + u64 address, size_t size) +{ + struct iommu_dev_data *dev_data; + int ret = 0; + ioasid_t pasid = IOMMU_NO_PASID; + bool gn = false; + + if (pdom_is_v2_pgtbl_mode(domain)) { + gn = true; + ret = domain_flush_pages_v2(domain, address, size); + } else { + ret = domain_flush_pages_v1(domain, address, size); + } + list_for_each_entry(dev_data, &domain->dev_list, list) { if (!dev_data->ats_enabled) @@ -1551,6 +1573,29 @@ static void amd_iommu_domain_flush_all(struct protection_domain *domain) CMD_INV_IOMMU_ALL_PAGES_ADDRESS); } +void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data, + ioasid_t pasid, u64 address, size_t size) +{ + struct iommu_cmd cmd; + struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); + + build_inv_iommu_pages(&cmd, address, size, + dev_data->gcr3_info.domid, pasid, true); + iommu_queue_command(iommu, &cmd); + + if (dev_data->ats_enabled) + device_flush_iotlb(dev_data, address, size, pasid, true); + + iommu_completion_wait(iommu); +} + +void amd_iommu_dev_flush_pasid_all(struct iommu_dev_data *dev_data, + ioasid_t pasid) +{ + amd_iommu_dev_flush_pasid_pages(dev_data, 0, + CMD_INV_IOMMU_ALL_PAGES_ADDRESS, pasid); +} + void amd_iommu_domain_flush_complete(struct protection_domain *domain) { int i; @@ -1592,6 +1637,49 @@ static void domain_flush_devices(struct protection_domain *domain) device_flush_dte(dev_data); } +static void update_device_table(struct protection_domain *domain) +{ + struct iommu_dev_data *dev_data; + + list_for_each_entry(dev_data, &domain->dev_list, list) { + struct amd_iommu *iommu = rlookup_amd_iommu(dev_data->dev); + + set_dte_entry(iommu, dev_data); + clone_aliases(iommu, dev_data->dev); + } +} + +void amd_iommu_update_and_flush_device_table(struct protection_domain *domain) +{ + update_device_table(domain); + domain_flush_devices(domain); +} + +void amd_iommu_domain_update(struct protection_domain *domain) +{ + /* Update device table */ + amd_iommu_update_and_flush_device_table(domain); + + /* Flush domain TLB(s) and wait for completion */ + amd_iommu_domain_flush_all(domain); +} + +int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid, + int status, int tag) +{ + struct iommu_dev_data *dev_data; + struct amd_iommu *iommu; + struct iommu_cmd cmd; + + dev_data = dev_iommu_priv_get(&pdev->dev); + iommu = get_amd_iommu_from_dev(&pdev->dev); + + build_complete_ppr(&cmd, dev_data->devid, pasid, status, + tag, dev_data->pri_tlp); + + return iommu_queue_command(iommu, &cmd); +} + /**************************************************************************** * * The next functions belong to the domain allocation. A domain is @@ -1656,16 +1744,22 @@ static void free_gcr3_tbl_level2(u64 *tbl) } } -static void free_gcr3_table(struct protection_domain *domain) +static void free_gcr3_table(struct gcr3_tbl_info *gcr3_info) { - if (domain->glx == 2) - free_gcr3_tbl_level2(domain->gcr3_tbl); - else if (domain->glx == 1) - free_gcr3_tbl_level1(domain->gcr3_tbl); + if (gcr3_info->glx == 2) + free_gcr3_tbl_level2(gcr3_info->gcr3_tbl); + else if (gcr3_info->glx == 1) + free_gcr3_tbl_level1(gcr3_info->gcr3_tbl); else - BUG_ON(domain->glx != 0); + WARN_ON_ONCE(gcr3_info->glx != 0); - free_page((unsigned long)domain->gcr3_tbl); + gcr3_info->glx = 0; + + /* Free per device domain ID */ + domain_id_free(gcr3_info->domid); + + free_page((unsigned long)gcr3_info->gcr3_tbl); + gcr3_info->gcr3_tbl = NULL; } /* @@ -1684,33 +1778,133 @@ static int get_gcr3_levels(int pasids) return levels ? (DIV_ROUND_UP(levels, 9) - 1) : levels; } -/* Note: This function expects iommu_domain->lock to be held prior calling the function. */ -static int setup_gcr3_table(struct protection_domain *domain, int pasids) +static int setup_gcr3_table(struct gcr3_tbl_info *gcr3_info, + struct amd_iommu *iommu, int pasids) { int levels = get_gcr3_levels(pasids); + int nid = iommu ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE; if (levels > amd_iommu_max_glx_val) return -EINVAL; - domain->gcr3_tbl = alloc_pgtable_page(domain->nid, GFP_ATOMIC); - if (domain->gcr3_tbl == NULL) + if (gcr3_info->gcr3_tbl) + return -EBUSY; + + /* Allocate per device domain ID */ + gcr3_info->domid = domain_id_alloc(); + + gcr3_info->gcr3_tbl = alloc_pgtable_page(nid, GFP_ATOMIC); + if (gcr3_info->gcr3_tbl == NULL) { + domain_id_free(gcr3_info->domid); return -ENOMEM; + } - domain->glx = levels; - domain->flags |= PD_IOMMUV2_MASK; - - amd_iommu_domain_update(domain); + gcr3_info->glx = levels; return 0; } -static void set_dte_entry(struct amd_iommu *iommu, u16 devid, - struct protection_domain *domain, bool ats, bool ppr) +static u64 *__get_gcr3_pte(struct gcr3_tbl_info *gcr3_info, + ioasid_t pasid, bool alloc) +{ + int index; + u64 *pte; + u64 *root = gcr3_info->gcr3_tbl; + int level = gcr3_info->glx; + + while (true) { + + index = (pasid >> (9 * level)) & 0x1ff; + pte = &root[index]; + + if (level == 0) + break; + + if (!(*pte & GCR3_VALID)) { + if (!alloc) + return NULL; + + root = (void *)get_zeroed_page(GFP_ATOMIC); + if (root == NULL) + return NULL; + + *pte = iommu_virt_to_phys(root) | GCR3_VALID; + } + + root = iommu_phys_to_virt(*pte & PAGE_MASK); + + level -= 1; + } + + return pte; +} + +static int update_gcr3(struct iommu_dev_data *dev_data, + ioasid_t pasid, unsigned long gcr3, bool set) +{ + struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; + u64 *pte; + + pte = __get_gcr3_pte(gcr3_info, pasid, true); + if (pte == NULL) + return -ENOMEM; + + if (set) + *pte = (gcr3 & PAGE_MASK) | GCR3_VALID; + else + *pte = 0; + + amd_iommu_dev_flush_pasid_all(dev_data, pasid); + return 0; +} + +int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid, + unsigned long gcr3) +{ + struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; + int ret; + + iommu_group_mutex_assert(dev_data->dev); + + ret = update_gcr3(dev_data, pasid, gcr3, true); + if (ret) + return ret; + + gcr3_info->pasid_cnt++; + return ret; +} + +int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid) +{ + struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; + int ret; + + iommu_group_mutex_assert(dev_data->dev); + + ret = update_gcr3(dev_data, pasid, 0, false); + if (ret) + return ret; + + gcr3_info->pasid_cnt--; + return ret; +} + +static void set_dte_entry(struct amd_iommu *iommu, + struct iommu_dev_data *dev_data) { u64 pte_root = 0; u64 flags = 0; u32 old_domid; + u16 devid = dev_data->devid; + u16 domid; + struct protection_domain *domain = dev_data->domain; struct dev_table_entry *dev_table = get_dev_table(iommu); + struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; + + if (gcr3_info && gcr3_info->gcr3_tbl) + domid = dev_data->gcr3_info.domid; + else + domid = domain->id; if (domain->iop.mode != PAGE_MODE_NONE) pte_root = iommu_virt_to_phys(domain->iop.root); @@ -1724,23 +1918,23 @@ static void set_dte_entry(struct amd_iommu *iommu, u16 devid, * When SNP is enabled, Only set TV bit when IOMMU * page translation is in use. */ - if (!amd_iommu_snp_en || (domain->id != 0)) + if (!amd_iommu_snp_en || (domid != 0)) pte_root |= DTE_FLAG_TV; flags = dev_table[devid].data[1]; - if (ats) + if (dev_data->ats_enabled) flags |= DTE_FLAG_IOTLB; - if (ppr) + if (dev_data->ppr) pte_root |= 1ULL << DEV_ENTRY_PPR; if (domain->dirty_tracking) pte_root |= DTE_FLAG_HAD; - if (domain->flags & PD_IOMMUV2_MASK) { - u64 gcr3 = iommu_virt_to_phys(domain->gcr3_tbl); - u64 glx = domain->glx; + if (gcr3_info && gcr3_info->gcr3_tbl) { + u64 gcr3 = iommu_virt_to_phys(gcr3_info->gcr3_tbl); + u64 glx = gcr3_info->glx; u64 tmp; pte_root |= DTE_FLAG_GV; @@ -1768,12 +1962,13 @@ static void set_dte_entry(struct amd_iommu *iommu, u16 devid, ((u64)GUEST_PGTABLE_5_LEVEL << DTE_GPT_LEVEL_SHIFT); } - if (domain->flags & PD_GIOV_MASK) + /* GIOV is supported with V2 page table mode only */ + if (pdom_is_v2_pgtbl_mode(domain)) pte_root |= DTE_FLAG_GIOV; } flags &= ~DEV_DOMID_MASK; - flags |= domain->id; + flags |= domid; old_domid = dev_table[devid].data[1] & DEV_DOMID_MASK; dev_table[devid].data[1] = flags; @@ -1804,16 +1999,11 @@ static void clear_dte_entry(struct amd_iommu *iommu, u16 devid) amd_iommu_apply_erratum_63(iommu, devid); } -static void do_attach(struct iommu_dev_data *dev_data, - struct protection_domain *domain) +static int do_attach(struct iommu_dev_data *dev_data, + struct protection_domain *domain) { - struct amd_iommu *iommu; - bool ats; - - iommu = rlookup_amd_iommu(dev_data->dev); - if (!iommu) - return; - ats = dev_data->ats_enabled; + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); + int ret = 0; /* Update data structures */ dev_data->domain = domain; @@ -1827,22 +2017,40 @@ static void do_attach(struct iommu_dev_data *dev_data, domain->dev_iommu[iommu->index] += 1; domain->dev_cnt += 1; + /* Init GCR3 table and update device table */ + if (domain->pd_mode == PD_MODE_V2) { + /* By default, setup GCR3 table to support single PASID */ + ret = setup_gcr3_table(&dev_data->gcr3_info, iommu, 1); + if (ret) + return ret; + + ret = update_gcr3(dev_data, 0, + iommu_virt_to_phys(domain->iop.pgd), true); + if (ret) { + free_gcr3_table(&dev_data->gcr3_info); + return ret; + } + } + /* Update device table */ - set_dte_entry(iommu, dev_data->devid, domain, - ats, dev_data->ppr); + set_dte_entry(iommu, dev_data); clone_aliases(iommu, dev_data->dev); device_flush_dte(dev_data); + + return ret; } static void do_detach(struct iommu_dev_data *dev_data) { struct protection_domain *domain = dev_data->domain; - struct amd_iommu *iommu; + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); - iommu = rlookup_amd_iommu(dev_data->dev); - if (!iommu) - return; + /* Clear GCR3 table */ + if (domain->pd_mode == PD_MODE_V2) { + update_gcr3(dev_data, 0, 0, false); + free_gcr3_table(&dev_data->gcr3_info); + } /* Update data structures */ dev_data->domain = NULL; @@ -1886,7 +2094,7 @@ static int attach_device(struct device *dev, if (dev_is_pci(dev)) pdev_enable_caps(to_pci_dev(dev)); - do_attach(dev_data, domain); + ret = do_attach(dev_data, domain); out: spin_unlock(&dev_data->lock); @@ -1954,8 +2162,7 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) ret = iommu_init_device(iommu, dev); if (ret) { - if (ret != -ENOTSUPP) - dev_err(dev, "Failed to initialize - trying to proceed anyway\n"); + dev_err(dev, "Failed to initialize - trying to proceed anyway\n"); iommu_dev = ERR_PTR(ret); iommu_ignore_device(iommu, dev); } else { @@ -1998,42 +2205,6 @@ static struct iommu_group *amd_iommu_device_group(struct device *dev) return acpihid_device_group(dev); } -/***************************************************************************** - * - * The next functions belong to the dma_ops mapping/unmapping code. - * - *****************************************************************************/ - -static void update_device_table(struct protection_domain *domain) -{ - struct iommu_dev_data *dev_data; - - list_for_each_entry(dev_data, &domain->dev_list, list) { - struct amd_iommu *iommu = rlookup_amd_iommu(dev_data->dev); - - if (!iommu) - continue; - set_dte_entry(iommu, dev_data->devid, domain, - dev_data->ats_enabled, dev_data->ppr); - clone_aliases(iommu, dev_data->dev); - } -} - -void amd_iommu_update_and_flush_device_table(struct protection_domain *domain) -{ - update_device_table(domain); - domain_flush_devices(domain); -} - -void amd_iommu_domain_update(struct protection_domain *domain) -{ - /* Update device table */ - amd_iommu_update_and_flush_device_table(domain); - - /* Flush domain TLB(s) and wait for completion */ - amd_iommu_domain_flush_all(domain); -} - /***************************************************************************** * * The following functions belong to the exported interface of AMD IOMMU @@ -2070,9 +2241,6 @@ static void protection_domain_free(struct protection_domain *domain) if (domain->iop.pgtbl_cfg.tlb) free_io_pgtable_ops(&domain->iop.iop.ops); - if (domain->flags & PD_IOMMUV2_MASK) - free_gcr3_table(domain); - if (domain->iop.root) free_page((unsigned long)domain->iop.root); @@ -2094,19 +2262,16 @@ static int protection_domain_init_v1(struct protection_domain *domain, int mode) return -ENOMEM; } + domain->pd_mode = PD_MODE_V1; amd_iommu_domain_set_pgtable(domain, pt_root, mode); return 0; } -static int protection_domain_init_v2(struct protection_domain *domain) +static int protection_domain_init_v2(struct protection_domain *pdom) { - domain->flags |= PD_GIOV_MASK; - - domain->domain.pgsize_bitmap = AMD_IOMMU_PGSIZES_V2; - - if (setup_gcr3_table(domain, 1)) - return -ENOMEM; + pdom->pd_mode = PD_MODE_V2; + pdom->domain.pgsize_bitmap = AMD_IOMMU_PGSIZES_V2; return 0; } @@ -2194,11 +2359,8 @@ static struct iommu_domain *do_iommu_domain_alloc(unsigned int type, struct protection_domain *domain; struct amd_iommu *iommu = NULL; - if (dev) { - iommu = rlookup_amd_iommu(dev); - if (!iommu) - return ERR_PTR(-ENODEV); - } + if (dev) + iommu = get_amd_iommu_from_dev(dev); /* * Since DTE[Mode]=0 is prohibited on SNP-enabled system, @@ -2279,7 +2441,7 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, { struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); struct protection_domain *domain = to_pdomain(dom); - struct amd_iommu *iommu = rlookup_amd_iommu(dev); + struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); int ret; /* @@ -2337,7 +2499,7 @@ static int amd_iommu_map_pages(struct iommu_domain *dom, unsigned long iova, int prot = 0; int ret = -EINVAL; - if ((amd_iommu_pgtable == AMD_IOMMU_V1) && + if ((domain->pd_mode == PD_MODE_V1) && (domain->iop.mode == PAGE_MODE_NONE)) return -EINVAL; @@ -2383,7 +2545,7 @@ static size_t amd_iommu_unmap_pages(struct iommu_domain *dom, unsigned long iova struct io_pgtable_ops *ops = &domain->iop.iop.ops; size_t r; - if ((amd_iommu_pgtable == AMD_IOMMU_V1) && + if ((domain->pd_mode == PD_MODE_V1) && (domain->iop.mode == PAGE_MODE_NONE)) return 0; @@ -2418,7 +2580,7 @@ static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap) case IOMMU_CAP_DEFERRED_FLUSH: return true; case IOMMU_CAP_DIRTY_TRACKING: { - struct amd_iommu *iommu = rlookup_amd_iommu(dev); + struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); return amd_iommu_hd_support(iommu); } @@ -2447,9 +2609,7 @@ static int amd_iommu_set_dirty_tracking(struct iommu_domain *domain, } list_for_each_entry(dev_data, &pdomain->dev_list, list) { - iommu = rlookup_amd_iommu(dev_data->dev); - if (!iommu) - continue; + iommu = get_amd_iommu_from_dev_data(dev_data); dev_table = get_dev_table(iommu); pte_root = dev_table[dev_data->devid].data[0]; @@ -2509,9 +2669,7 @@ static void amd_iommu_get_resv_regions(struct device *dev, return; devid = PCI_SBDF_TO_DEVID(sbdf); - iommu = rlookup_amd_iommu(dev); - if (!iommu) - return; + iommu = get_amd_iommu_from_dev(dev); pci_seg = iommu->pci_seg; list_for_each_entry(entry, &pci_seg->unity_map, list) { @@ -2645,216 +2803,6 @@ const struct iommu_ops amd_iommu_ops = { } }; -static int __flush_pasid(struct protection_domain *domain, u32 pasid, - u64 address, size_t size) -{ - struct iommu_dev_data *dev_data; - struct iommu_cmd cmd; - int i, ret; - - if (!(domain->flags & PD_IOMMUV2_MASK)) - return -EINVAL; - - build_inv_iommu_pages(&cmd, address, size, domain->id, pasid, true); - - /* - * IOMMU TLB needs to be flushed before Device TLB to - * prevent device TLB refill from IOMMU TLB - */ - for (i = 0; i < amd_iommu_get_num_iommus(); ++i) { - if (domain->dev_iommu[i] == 0) - continue; - - ret = iommu_queue_command(amd_iommus[i], &cmd); - if (ret != 0) - goto out; - } - - /* Wait until IOMMU TLB flushes are complete */ - amd_iommu_domain_flush_complete(domain); - - /* Now flush device TLBs */ - list_for_each_entry(dev_data, &domain->dev_list, list) { - struct amd_iommu *iommu; - int qdep; - - /* - There might be non-IOMMUv2 capable devices in an IOMMUv2 - * domain. - */ - if (!dev_data->ats_enabled) - continue; - - qdep = dev_data->ats_qdep; - iommu = rlookup_amd_iommu(dev_data->dev); - if (!iommu) - continue; - build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, - address, size, pasid, true); - - ret = iommu_queue_command(iommu, &cmd); - if (ret != 0) - goto out; - } - - /* Wait until all device TLBs are flushed */ - amd_iommu_domain_flush_complete(domain); - - ret = 0; - -out: - - return ret; -} - -static int __amd_iommu_flush_page(struct protection_domain *domain, u32 pasid, - u64 address) -{ - return __flush_pasid(domain, pasid, address, PAGE_SIZE); -} - -int amd_iommu_flush_page(struct iommu_domain *dom, u32 pasid, - u64 address) -{ - struct protection_domain *domain = to_pdomain(dom); - unsigned long flags; - int ret; - - spin_lock_irqsave(&domain->lock, flags); - ret = __amd_iommu_flush_page(domain, pasid, address); - spin_unlock_irqrestore(&domain->lock, flags); - - return ret; -} - -static int __amd_iommu_flush_tlb(struct protection_domain *domain, u32 pasid) -{ - return __flush_pasid(domain, pasid, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS); -} - -int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid) -{ - struct protection_domain *domain = to_pdomain(dom); - unsigned long flags; - int ret; - - spin_lock_irqsave(&domain->lock, flags); - ret = __amd_iommu_flush_tlb(domain, pasid); - spin_unlock_irqrestore(&domain->lock, flags); - - return ret; -} - -static u64 *__get_gcr3_pte(u64 *root, int level, u32 pasid, bool alloc) -{ - int index; - u64 *pte; - - while (true) { - - index = (pasid >> (9 * level)) & 0x1ff; - pte = &root[index]; - - if (level == 0) - break; - - if (!(*pte & GCR3_VALID)) { - if (!alloc) - return NULL; - - root = (void *)get_zeroed_page(GFP_ATOMIC); - if (root == NULL) - return NULL; - - *pte = iommu_virt_to_phys(root) | GCR3_VALID; - } - - root = iommu_phys_to_virt(*pte & PAGE_MASK); - - level -= 1; - } - - return pte; -} - -static int __set_gcr3(struct protection_domain *domain, u32 pasid, - unsigned long cr3) -{ - u64 *pte; - - if (domain->iop.mode != PAGE_MODE_NONE) - return -EINVAL; - - pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true); - if (pte == NULL) - return -ENOMEM; - - *pte = (cr3 & PAGE_MASK) | GCR3_VALID; - - return __amd_iommu_flush_tlb(domain, pasid); -} - -static int __clear_gcr3(struct protection_domain *domain, u32 pasid) -{ - u64 *pte; - - if (domain->iop.mode != PAGE_MODE_NONE) - return -EINVAL; - - pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false); - if (pte == NULL) - return 0; - - *pte = 0; - - return __amd_iommu_flush_tlb(domain, pasid); -} - -int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, u32 pasid, - unsigned long cr3) -{ - struct protection_domain *domain = to_pdomain(dom); - unsigned long flags; - int ret; - - spin_lock_irqsave(&domain->lock, flags); - ret = __set_gcr3(domain, pasid, cr3); - spin_unlock_irqrestore(&domain->lock, flags); - - return ret; -} - -int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, u32 pasid) -{ - struct protection_domain *domain = to_pdomain(dom); - unsigned long flags; - int ret; - - spin_lock_irqsave(&domain->lock, flags); - ret = __clear_gcr3(domain, pasid); - spin_unlock_irqrestore(&domain->lock, flags); - - return ret; -} - -int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid, - int status, int tag) -{ - struct iommu_dev_data *dev_data; - struct amd_iommu *iommu; - struct iommu_cmd cmd; - - dev_data = dev_iommu_priv_get(&pdev->dev); - iommu = rlookup_amd_iommu(&pdev->dev); - if (!iommu) - return -ENODEV; - - build_complete_ppr(&cmd, dev_data->devid, pasid, status, - tag, dev_data->pri_tlp); - - return iommu_queue_command(iommu, &cmd); -} - #ifdef CONFIG_IRQ_REMAP /***************************************************************************** diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index ef3ee95706da..eb1e62cd499a 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -779,7 +779,8 @@ static void apple_dart_domain_free(struct iommu_domain *domain) kfree(dart_domain); } -static int apple_dart_of_xlate(struct device *dev, struct of_phandle_args *args) +static int apple_dart_of_xlate(struct device *dev, + const struct of_phandle_args *args) { struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); struct platform_device *iommu_pdev = of_find_device_by_node(args->np); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 4a27fbdb2d84..2cd433a9c8a0 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -10,7 +10,6 @@ #include #include "arm-smmu-v3.h" -#include "../../iommu-sva.h" #include "../../io-pgtable-arm.h" struct arm_smmu_mmu_notifier { @@ -364,7 +363,13 @@ static int __arm_smmu_sva_bind(struct device *dev, ioasid_t pasid, struct arm_smmu_bond *bond; struct arm_smmu_master *master = dev_iommu_priv_get(dev); struct iommu_domain *domain = iommu_get_domain_for_dev(dev); - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_domain *smmu_domain; + + if (!(domain->type & __IOMMU_DOMAIN_PAGING)) + return -ENODEV; + smmu_domain = to_smmu_domain(domain); + if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1) + return -ENODEV; if (!master || !master->sva_enabled) return -ENODEV; @@ -470,7 +475,6 @@ bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master) static int arm_smmu_master_sva_enable_iopf(struct arm_smmu_master *master) { - int ret; struct device *dev = master->dev; /* @@ -483,16 +487,7 @@ static int arm_smmu_master_sva_enable_iopf(struct arm_smmu_master *master) if (!master->iopf_enabled) return -EINVAL; - ret = iopf_queue_add_device(master->smmu->evtq.iopf, dev); - if (ret) - return ret; - - ret = iommu_register_device_fault_handler(dev, iommu_queue_iopf, dev); - if (ret) { - iopf_queue_remove_device(master->smmu->evtq.iopf, dev); - return ret; - } - return 0; + return iopf_queue_add_device(master->smmu->evtq.iopf, dev); } static void arm_smmu_master_sva_disable_iopf(struct arm_smmu_master *master) @@ -502,7 +497,6 @@ static void arm_smmu_master_sva_disable_iopf(struct arm_smmu_master *master) if (!master->iopf_enabled) return; - iommu_unregister_device_fault_handler(dev); iopf_queue_remove_device(master->smmu->evtq.iopf, dev); } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index a74a509bcd63..5ed036225e69 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -29,7 +29,6 @@ #include "arm-smmu-v3.h" #include "../../dma-iommu.h" -#include "../../iommu-sva.h" static bool disable_bypass = true; module_param(disable_bypass, bool, 0444); @@ -48,6 +47,9 @@ enum arm_smmu_msi_index { ARM_SMMU_MAX_MSIS, }; +static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, + ioasid_t sid); + static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = { [EVTQ_MSI_INDEX] = { ARM_SMMU_EVTQ_IRQ_CFG0, @@ -86,6 +88,9 @@ static struct arm_smmu_option_prop arm_smmu_options[] = { { 0, NULL}, }; +static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain, + struct arm_smmu_device *smmu); + static void parse_driver_options(struct arm_smmu_device *smmu) { int i = 0; @@ -921,31 +926,29 @@ static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu, return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, true); } -static int arm_smmu_page_response(struct device *dev, - struct iommu_fault_event *unused, - struct iommu_page_response *resp) +static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused, + struct iommu_page_response *resp) { struct arm_smmu_cmdq_ent cmd = {0}; struct arm_smmu_master *master = dev_iommu_priv_get(dev); int sid = master->streams[0].id; - if (master->stall_enabled) { - cmd.opcode = CMDQ_OP_RESUME; - cmd.resume.sid = sid; - cmd.resume.stag = resp->grpid; - switch (resp->code) { - case IOMMU_PAGE_RESP_INVALID: - case IOMMU_PAGE_RESP_FAILURE: - cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT; - break; - case IOMMU_PAGE_RESP_SUCCESS: - cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY; - break; - default: - return -EINVAL; - } - } else { - return -ENODEV; + if (WARN_ON(!master->stall_enabled)) + return; + + cmd.opcode = CMDQ_OP_RESUME; + cmd.resume.sid = sid; + cmd.resume.stag = resp->grpid; + switch (resp->code) { + case IOMMU_PAGE_RESP_INVALID: + case IOMMU_PAGE_RESP_FAILURE: + cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT; + break; + case IOMMU_PAGE_RESP_SUCCESS: + cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY; + break; + default: + break; } arm_smmu_cmdq_issue_cmd(master->smmu, &cmd); @@ -955,8 +958,6 @@ static int arm_smmu_page_response(struct device *dev, * terminated... at some point in the future. PRI_RESP is fire and * forget. */ - - return 0; } /* Context descriptor manipulation functions */ @@ -971,6 +972,199 @@ void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid) arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); } +/* + * Based on the value of ent report which bits of the STE the HW will access. It + * would be nice if this was complete according to the spec, but minimally it + * has to capture the bits this driver uses. + */ +static void arm_smmu_get_ste_used(const struct arm_smmu_ste *ent, + struct arm_smmu_ste *used_bits) +{ + unsigned int cfg = FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(ent->data[0])); + + used_bits->data[0] = cpu_to_le64(STRTAB_STE_0_V); + if (!(ent->data[0] & cpu_to_le64(STRTAB_STE_0_V))) + return; + + used_bits->data[0] |= cpu_to_le64(STRTAB_STE_0_CFG); + + /* S1 translates */ + if (cfg & BIT(0)) { + used_bits->data[0] |= cpu_to_le64(STRTAB_STE_0_S1FMT | + STRTAB_STE_0_S1CTXPTR_MASK | + STRTAB_STE_0_S1CDMAX); + used_bits->data[1] |= + cpu_to_le64(STRTAB_STE_1_S1DSS | STRTAB_STE_1_S1CIR | + STRTAB_STE_1_S1COR | STRTAB_STE_1_S1CSH | + STRTAB_STE_1_S1STALLD | STRTAB_STE_1_STRW | + STRTAB_STE_1_EATS); + used_bits->data[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID); + } + + /* S2 translates */ + if (cfg & BIT(1)) { + used_bits->data[1] |= + cpu_to_le64(STRTAB_STE_1_EATS | STRTAB_STE_1_SHCFG); + used_bits->data[2] |= + cpu_to_le64(STRTAB_STE_2_S2VMID | STRTAB_STE_2_VTCR | + STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2ENDI | + STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2R); + used_bits->data[3] |= cpu_to_le64(STRTAB_STE_3_S2TTB_MASK); + } + + if (cfg == STRTAB_STE_0_CFG_BYPASS) + used_bits->data[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG); +} + +/* + * Figure out if we can do a hitless update of entry to become target. Returns a + * bit mask where 1 indicates that qword needs to be set disruptively. + * unused_update is an intermediate value of entry that has unused bits set to + * their new values. + */ +static u8 arm_smmu_entry_qword_diff(const struct arm_smmu_ste *entry, + const struct arm_smmu_ste *target, + struct arm_smmu_ste *unused_update) +{ + struct arm_smmu_ste target_used = {}; + struct arm_smmu_ste cur_used = {}; + u8 used_qword_diff = 0; + unsigned int i; + + arm_smmu_get_ste_used(entry, &cur_used); + arm_smmu_get_ste_used(target, &target_used); + + for (i = 0; i != ARRAY_SIZE(target_used.data); i++) { + /* + * Check that masks are up to date, the make functions are not + * allowed to set a bit to 1 if the used function doesn't say it + * is used. + */ + WARN_ON_ONCE(target->data[i] & ~target_used.data[i]); + + /* Bits can change because they are not currently being used */ + unused_update->data[i] = (entry->data[i] & cur_used.data[i]) | + (target->data[i] & ~cur_used.data[i]); + /* + * Each bit indicates that a used bit in a qword needs to be + * changed after unused_update is applied. + */ + if ((unused_update->data[i] & target_used.data[i]) != + target->data[i]) + used_qword_diff |= 1 << i; + } + return used_qword_diff; +} + +static bool entry_set(struct arm_smmu_device *smmu, ioasid_t sid, + struct arm_smmu_ste *entry, + const struct arm_smmu_ste *target, unsigned int start, + unsigned int len) +{ + bool changed = false; + unsigned int i; + + for (i = start; len != 0; len--, i++) { + if (entry->data[i] != target->data[i]) { + WRITE_ONCE(entry->data[i], target->data[i]); + changed = true; + } + } + + if (changed) + arm_smmu_sync_ste_for_sid(smmu, sid); + return changed; +} + +/* + * Update the STE/CD to the target configuration. The transition from the + * current entry to the target entry takes place over multiple steps that + * attempts to make the transition hitless if possible. This function takes care + * not to create a situation where the HW can perceive a corrupted entry. HW is + * only required to have a 64 bit atomicity with stores from the CPU, while + * entries are many 64 bit values big. + * + * The difference between the current value and the target value is analyzed to + * determine which of three updates are required - disruptive, hitless or no + * change. + * + * In the most general disruptive case we can make any update in three steps: + * - Disrupting the entry (V=0) + * - Fill now unused qwords, execpt qword 0 which contains V + * - Make qword 0 have the final value and valid (V=1) with a single 64 + * bit store + * + * However this disrupts the HW while it is happening. There are several + * interesting cases where a STE/CD can be updated without disturbing the HW + * because only a small number of bits are changing (S1DSS, CONFIG, etc) or + * because the used bits don't intersect. We can detect this by calculating how + * many 64 bit values need update after adjusting the unused bits and skip the + * V=0 process. This relies on the IGNORED behavior described in the + * specification. + */ +static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, + struct arm_smmu_ste *entry, + const struct arm_smmu_ste *target) +{ + unsigned int num_entry_qwords = ARRAY_SIZE(target->data); + struct arm_smmu_device *smmu = master->smmu; + struct arm_smmu_ste unused_update; + u8 used_qword_diff; + + used_qword_diff = + arm_smmu_entry_qword_diff(entry, target, &unused_update); + if (hweight8(used_qword_diff) == 1) { + /* + * Only one qword needs its used bits to be changed. This is a + * hitless update, update all bits the current STE is ignoring + * to their new values, then update a single "critical qword" to + * change the STE and finally 0 out any bits that are now unused + * in the target configuration. + */ + unsigned int critical_qword_index = ffs(used_qword_diff) - 1; + + /* + * Skip writing unused bits in the critical qword since we'll be + * writing it in the next step anyways. This can save a sync + * when the only change is in that qword. + */ + unused_update.data[critical_qword_index] = + entry->data[critical_qword_index]; + entry_set(smmu, sid, entry, &unused_update, 0, num_entry_qwords); + entry_set(smmu, sid, entry, target, critical_qword_index, 1); + entry_set(smmu, sid, entry, target, 0, num_entry_qwords); + } else if (used_qword_diff) { + /* + * At least two qwords need their inuse bits to be changed. This + * requires a breaking update, zero the V bit, write all qwords + * but 0, then set qword 0 + */ + unused_update.data[0] = entry->data[0] & (~STRTAB_STE_0_V); + entry_set(smmu, sid, entry, &unused_update, 0, 1); + entry_set(smmu, sid, entry, target, 1, num_entry_qwords - 1); + entry_set(smmu, sid, entry, target, 0, 1); + } else { + /* + * No inuse bit changed. Sanity check that all unused bits are 0 + * in the entry. The target was already sanity checked by + * compute_qword_diff(). + */ + WARN_ON_ONCE( + entry_set(smmu, sid, entry, target, 0, num_entry_qwords)); + } + + /* It's likely that we'll want to use the new STE soon */ + if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) { + struct arm_smmu_cmdq_ent + prefetch_cmd = { .opcode = CMDQ_OP_PREFETCH_CFG, + .prefetch = { + .sid = sid, + } }; + + arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); + } +} + static void arm_smmu_sync_cd(struct arm_smmu_master *master, int ssid, bool leaf) { @@ -1251,158 +1445,131 @@ static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid) arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); } -static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, - struct arm_smmu_ste *dst) +static void arm_smmu_make_abort_ste(struct arm_smmu_ste *target) { - /* - * This is hideously complicated, but we only really care about - * three cases at the moment: - * - * 1. Invalid (all zero) -> bypass/fault (init) - * 2. Bypass/fault -> translation/bypass (attach) - * 3. Translation/bypass -> bypass/fault (detach) - * - * Given that we can't update the STE atomically and the SMMU - * doesn't read the thing in a defined order, that leaves us - * with the following maintenance requirements: - * - * 1. Update Config, return (init time STEs aren't live) - * 2. Write everything apart from dword 0, sync, write dword 0, sync - * 3. Update Config, sync - */ - u64 val = le64_to_cpu(dst->data[0]); - bool ste_live = false; - struct arm_smmu_device *smmu = master->smmu; - struct arm_smmu_ctx_desc_cfg *cd_table = NULL; - struct arm_smmu_s2_cfg *s2_cfg = NULL; - struct arm_smmu_domain *smmu_domain = master->domain; - struct arm_smmu_cmdq_ent prefetch_cmd = { - .opcode = CMDQ_OP_PREFETCH_CFG, - .prefetch = { - .sid = sid, - }, - }; - - if (smmu_domain) { - switch (smmu_domain->stage) { - case ARM_SMMU_DOMAIN_S1: - cd_table = &master->cd_table; - break; - case ARM_SMMU_DOMAIN_S2: - s2_cfg = &smmu_domain->s2_cfg; - break; - default: - break; - } - } - - if (val & STRTAB_STE_0_V) { - switch (FIELD_GET(STRTAB_STE_0_CFG, val)) { - case STRTAB_STE_0_CFG_BYPASS: - break; - case STRTAB_STE_0_CFG_S1_TRANS: - case STRTAB_STE_0_CFG_S2_TRANS: - ste_live = true; - break; - case STRTAB_STE_0_CFG_ABORT: - BUG_ON(!disable_bypass); - break; - default: - BUG(); /* STE corruption */ - } - } - - /* Nuke the existing STE_0 value, as we're going to rewrite it */ - val = STRTAB_STE_0_V; - - /* Bypass/fault */ - if (!smmu_domain || !(cd_table || s2_cfg)) { - if (!smmu_domain && disable_bypass) - val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT); - else - val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS); - - dst->data[0] = cpu_to_le64(val); - dst->data[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG, - STRTAB_STE_1_SHCFG_INCOMING)); - dst->data[2] = 0; /* Nuke the VMID */ - /* - * The SMMU can perform negative caching, so we must sync - * the STE regardless of whether the old value was live. - */ - if (smmu) - arm_smmu_sync_ste_for_sid(smmu, sid); - return; - } - - if (cd_table) { - u64 strw = smmu->features & ARM_SMMU_FEAT_E2H ? - STRTAB_STE_1_STRW_EL2 : STRTAB_STE_1_STRW_NSEL1; - - BUG_ON(ste_live); - dst->data[1] = cpu_to_le64( - FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) | - FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) | - FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) | - FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) | - FIELD_PREP(STRTAB_STE_1_STRW, strw)); - - if (smmu->features & ARM_SMMU_FEAT_STALLS && - !master->stall_enabled) - dst->data[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD); - - val |= (cd_table->cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) | - FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) | - FIELD_PREP(STRTAB_STE_0_S1CDMAX, cd_table->s1cdmax) | - FIELD_PREP(STRTAB_STE_0_S1FMT, cd_table->s1fmt); - } - - if (s2_cfg) { - BUG_ON(ste_live); - dst->data[2] = cpu_to_le64( - FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) | - FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) | -#ifdef __BIG_ENDIAN - STRTAB_STE_2_S2ENDI | -#endif - STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 | - STRTAB_STE_2_S2R); - - dst->data[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK); - - val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS); - } - - if (master->ats_enabled) - dst->data[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS, - STRTAB_STE_1_EATS_TRANS)); - - arm_smmu_sync_ste_for_sid(smmu, sid); - /* See comment in arm_smmu_write_ctx_desc() */ - WRITE_ONCE(dst->data[0], cpu_to_le64(val)); - arm_smmu_sync_ste_for_sid(smmu, sid); - - /* It's likely that we'll want to use the new STE soon */ - if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) - arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); + memset(target, 0, sizeof(*target)); + target->data[0] = cpu_to_le64( + STRTAB_STE_0_V | + FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT)); } -static void arm_smmu_init_bypass_stes(struct arm_smmu_ste *strtab, - unsigned int nent, bool force) +static void arm_smmu_make_bypass_ste(struct arm_smmu_ste *target) +{ + memset(target, 0, sizeof(*target)); + target->data[0] = cpu_to_le64( + STRTAB_STE_0_V | + FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS)); + target->data[1] = cpu_to_le64( + FIELD_PREP(STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING)); +} + +static void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, + struct arm_smmu_master *master) +{ + struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table; + struct arm_smmu_device *smmu = master->smmu; + + memset(target, 0, sizeof(*target)); + target->data[0] = cpu_to_le64( + STRTAB_STE_0_V | + FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) | + FIELD_PREP(STRTAB_STE_0_S1FMT, cd_table->s1fmt) | + (cd_table->cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) | + FIELD_PREP(STRTAB_STE_0_S1CDMAX, cd_table->s1cdmax)); + + target->data[1] = cpu_to_le64( + FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) | + FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) | + FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) | + FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) | + ((smmu->features & ARM_SMMU_FEAT_STALLS && + !master->stall_enabled) ? + STRTAB_STE_1_S1STALLD : + 0) | + FIELD_PREP(STRTAB_STE_1_EATS, + master->ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0)); + + if (smmu->features & ARM_SMMU_FEAT_E2H) { + /* + * To support BTM the streamworld needs to match the + * configuration of the CPU so that the ASID broadcasts are + * properly matched. This means either S/NS-EL2-E2H (hypervisor) + * or NS-EL1 (guest). Since an SVA domain can be installed in a + * PASID this should always use a BTM compatible configuration + * if the HW supports it. + */ + target->data[1] |= cpu_to_le64( + FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_EL2)); + } else { + target->data[1] |= cpu_to_le64( + FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1)); + + /* + * VMID 0 is reserved for stage-2 bypass EL1 STEs, see + * arm_smmu_domain_alloc_id() + */ + target->data[2] = + cpu_to_le64(FIELD_PREP(STRTAB_STE_2_S2VMID, 0)); + } +} + +static void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, + struct arm_smmu_master *master, + struct arm_smmu_domain *smmu_domain) +{ + struct arm_smmu_s2_cfg *s2_cfg = &smmu_domain->s2_cfg; + const struct io_pgtable_cfg *pgtbl_cfg = + &io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops)->cfg; + typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr = + &pgtbl_cfg->arm_lpae_s2_cfg.vtcr; + u64 vtcr_val; + + memset(target, 0, sizeof(*target)); + target->data[0] = cpu_to_le64( + STRTAB_STE_0_V | + FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS)); + + target->data[1] = cpu_to_le64( + FIELD_PREP(STRTAB_STE_1_EATS, + master->ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0) | + FIELD_PREP(STRTAB_STE_1_SHCFG, + STRTAB_STE_1_SHCFG_INCOMING)); + + vtcr_val = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps); + target->data[2] = cpu_to_le64( + FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) | + FIELD_PREP(STRTAB_STE_2_VTCR, vtcr_val) | + STRTAB_STE_2_S2AA64 | +#ifdef __BIG_ENDIAN + STRTAB_STE_2_S2ENDI | +#endif + STRTAB_STE_2_S2PTW | + STRTAB_STE_2_S2R); + + target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s2_cfg.vttbr & + STRTAB_STE_3_S2TTB_MASK); +} + +/* + * This can safely directly manipulate the STE memory without a sync sequence + * because the STE table has not been installed in the SMMU yet. + */ +static void arm_smmu_init_initial_stes(struct arm_smmu_ste *strtab, + unsigned int nent) { unsigned int i; - u64 val = STRTAB_STE_0_V; - - if (disable_bypass && !force) - val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT); - else - val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS); for (i = 0; i < nent; ++i) { - strtab->data[0] = cpu_to_le64(val); - strtab->data[1] = cpu_to_le64(FIELD_PREP( - STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING)); - strtab->data[2] = 0; + if (disable_bypass) + arm_smmu_make_abort_ste(strtab); + else + arm_smmu_make_bypass_ste(strtab); strtab++; } } @@ -1430,7 +1597,7 @@ static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) return -ENOMEM; } - arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT, false); + arm_smmu_init_initial_stes(desc->l2ptr, 1 << STRTAB_SPLIT); arm_smmu_write_strtab_l1_desc(strtab, desc); return 0; } @@ -1460,27 +1627,19 @@ arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid) /* IRQ and event handlers */ static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) { - int ret; - u32 reason; + int ret = 0; u32 perm = 0; struct arm_smmu_master *master; bool ssid_valid = evt[0] & EVTQ_0_SSV; u32 sid = FIELD_GET(EVTQ_0_SID, evt[0]); - struct iommu_fault_event fault_evt = { }; + struct iopf_fault fault_evt = { }; struct iommu_fault *flt = &fault_evt.fault; switch (FIELD_GET(EVTQ_0_ID, evt[0])) { case EVT_ID_TRANSLATION_FAULT: - reason = IOMMU_FAULT_REASON_PTE_FETCH; - break; case EVT_ID_ADDR_SIZE_FAULT: - reason = IOMMU_FAULT_REASON_OOR_ADDRESS; - break; case EVT_ID_ACCESS_FAULT: - reason = IOMMU_FAULT_REASON_ACCESS; - break; case EVT_ID_PERMISSION_FAULT: - reason = IOMMU_FAULT_REASON_PERMISSION; break; default: return -EOPNOTSUPP; @@ -1490,6 +1649,9 @@ static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) if (evt[1] & EVTQ_1_S2) return -EFAULT; + if (!(evt[1] & EVTQ_1_STALL)) + return -EOPNOTSUPP; + if (evt[1] & EVTQ_1_RnW) perm |= IOMMU_FAULT_PERM_READ; else @@ -1501,32 +1663,17 @@ static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) if (evt[1] & EVTQ_1_PnU) perm |= IOMMU_FAULT_PERM_PRIV; - if (evt[1] & EVTQ_1_STALL) { - flt->type = IOMMU_FAULT_PAGE_REQ; - flt->prm = (struct iommu_fault_page_request) { - .flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE, - .grpid = FIELD_GET(EVTQ_1_STAG, evt[1]), - .perm = perm, - .addr = FIELD_GET(EVTQ_2_ADDR, evt[2]), - }; + flt->type = IOMMU_FAULT_PAGE_REQ; + flt->prm = (struct iommu_fault_page_request) { + .flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE, + .grpid = FIELD_GET(EVTQ_1_STAG, evt[1]), + .perm = perm, + .addr = FIELD_GET(EVTQ_2_ADDR, evt[2]), + }; - if (ssid_valid) { - flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; - flt->prm.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]); - } - } else { - flt->type = IOMMU_FAULT_DMA_UNRECOV; - flt->event = (struct iommu_fault_unrecoverable) { - .reason = reason, - .flags = IOMMU_FAULT_UNRECOV_ADDR_VALID, - .perm = perm, - .addr = FIELD_GET(EVTQ_2_ADDR, evt[2]), - }; - - if (ssid_valid) { - flt->event.flags |= IOMMU_FAULT_UNRECOV_PASID_VALID; - flt->event.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]); - } + if (ssid_valid) { + flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; + flt->prm.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]); } mutex_lock(&smmu->streams_mutex); @@ -1536,17 +1683,7 @@ static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt) goto out_unlock; } - ret = iommu_report_device_fault(master->dev, &fault_evt); - if (ret && flt->type == IOMMU_FAULT_PAGE_REQ) { - /* Nobody cared, abort the access */ - struct iommu_page_response resp = { - .pasid = flt->prm.pasid, - .grpid = flt->prm.grpid, - .code = IOMMU_PAGE_RESP_FAILURE, - }; - arm_smmu_page_response(master->dev, &fault_evt, &resp); - } - + iommu_report_device_fault(master->dev, &fault_evt); out_unlock: mutex_unlock(&smmu->streams_mutex); return ret; @@ -2025,15 +2162,15 @@ static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap) static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) { - struct arm_smmu_domain *smmu_domain; if (type == IOMMU_DOMAIN_SVA) return arm_smmu_sva_domain_alloc(); + return ERR_PTR(-EOPNOTSUPP); +} - if (type != IOMMU_DOMAIN_UNMANAGED && - type != IOMMU_DOMAIN_DMA && - type != IOMMU_DOMAIN_IDENTITY) - return NULL; +static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev) +{ + struct arm_smmu_domain *smmu_domain; /* * Allocate the domain and initialise some of its data structures. @@ -2042,13 +2179,23 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) */ smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL); if (!smmu_domain) - return NULL; + return ERR_PTR(-ENOMEM); mutex_init(&smmu_domain->init_mutex); INIT_LIST_HEAD(&smmu_domain->devices); spin_lock_init(&smmu_domain->devices_lock); INIT_LIST_HEAD(&smmu_domain->mmu_notifiers); + if (dev) { + struct arm_smmu_master *master = dev_iommu_priv_get(dev); + int ret; + + ret = arm_smmu_domain_finalise(smmu_domain, master->smmu); + if (ret) { + kfree(smmu_domain); + return ERR_PTR(ret); + } + } return &smmu_domain->domain; } @@ -2074,12 +2221,12 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) kfree(smmu_domain); } -static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain, +static int arm_smmu_domain_finalise_s1(struct arm_smmu_device *smmu, + struct arm_smmu_domain *smmu_domain, struct io_pgtable_cfg *pgtbl_cfg) { int ret; u32 asid; - struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_ctx_desc *cd = &smmu_domain->cd; typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr; @@ -2111,13 +2258,12 @@ out_unlock: return ret; } -static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain, +static int arm_smmu_domain_finalise_s2(struct arm_smmu_device *smmu, + struct arm_smmu_domain *smmu_domain, struct io_pgtable_cfg *pgtbl_cfg) { int vmid; - struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg; - typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr; /* Reserve VMID 0 for stage-2 bypass STEs */ vmid = ida_alloc_range(&smmu->vmid_map, 1, (1 << smmu->vmid_bits) - 1, @@ -2125,35 +2271,21 @@ static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain, if (vmid < 0) return vmid; - vtcr = &pgtbl_cfg->arm_lpae_s2_cfg.vtcr; cfg->vmid = (u16)vmid; - cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; - cfg->vtcr = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) | - FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) | - FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) | - FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) | - FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) | - FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) | - FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps); return 0; } -static int arm_smmu_domain_finalise(struct iommu_domain *domain) +static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain, + struct arm_smmu_device *smmu) { int ret; unsigned long ias, oas; enum io_pgtable_fmt fmt; struct io_pgtable_cfg pgtbl_cfg; struct io_pgtable_ops *pgtbl_ops; - int (*finalise_stage_fn)(struct arm_smmu_domain *, - struct io_pgtable_cfg *); - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct arm_smmu_device *smmu = smmu_domain->smmu; - - if (domain->type == IOMMU_DOMAIN_IDENTITY) { - smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS; - return 0; - } + int (*finalise_stage_fn)(struct arm_smmu_device *smmu, + struct arm_smmu_domain *smmu_domain, + struct io_pgtable_cfg *pgtbl_cfg); /* Restrict the stage to what we can actually support */ if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) @@ -2192,17 +2324,18 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) if (!pgtbl_ops) return -ENOMEM; - domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; - domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1; - domain->geometry.force_aperture = true; + smmu_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; + smmu_domain->domain.geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1; + smmu_domain->domain.geometry.force_aperture = true; - ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg); + ret = finalise_stage_fn(smmu, smmu_domain, &pgtbl_cfg); if (ret < 0) { free_io_pgtable_ops(pgtbl_ops); return ret; } smmu_domain->pgtbl_ops = pgtbl_ops; + smmu_domain->smmu = smmu; return 0; } @@ -2225,7 +2358,8 @@ arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid) } } -static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master) +static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master, + const struct arm_smmu_ste *target) { int i, j; struct arm_smmu_device *smmu = master->smmu; @@ -2242,7 +2376,7 @@ static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master) if (j < i) continue; - arm_smmu_write_strtab_ent(master, sid, step); + arm_smmu_write_ste(master, sid, step, target); } } @@ -2261,12 +2395,12 @@ static bool arm_smmu_ats_supported(struct arm_smmu_master *master) return dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev)); } -static void arm_smmu_enable_ats(struct arm_smmu_master *master) +static void arm_smmu_enable_ats(struct arm_smmu_master *master, + struct arm_smmu_domain *smmu_domain) { size_t stu; struct pci_dev *pdev; struct arm_smmu_device *smmu = master->smmu; - struct arm_smmu_domain *smmu_domain = master->domain; /* Don't enable ATS at the endpoint if it's not enabled in the STE */ if (!master->ats_enabled) @@ -2282,10 +2416,9 @@ static void arm_smmu_enable_ats(struct arm_smmu_master *master) dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu); } -static void arm_smmu_disable_ats(struct arm_smmu_master *master) +static void arm_smmu_disable_ats(struct arm_smmu_master *master, + struct arm_smmu_domain *smmu_domain) { - struct arm_smmu_domain *smmu_domain = master->domain; - if (!master->ats_enabled) return; @@ -2348,35 +2481,28 @@ static void arm_smmu_disable_pasid(struct arm_smmu_master *master) static void arm_smmu_detach_dev(struct arm_smmu_master *master) { + struct iommu_domain *domain = iommu_get_domain_for_dev(master->dev); + struct arm_smmu_domain *smmu_domain; unsigned long flags; - struct arm_smmu_domain *smmu_domain = master->domain; - if (!smmu_domain) + if (!domain || !(domain->type & __IOMMU_DOMAIN_PAGING)) return; - arm_smmu_disable_ats(master); + smmu_domain = to_smmu_domain(domain); + arm_smmu_disable_ats(master, smmu_domain); spin_lock_irqsave(&smmu_domain->devices_lock, flags); - list_del(&master->domain_head); + list_del_init(&master->domain_head); spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); - master->domain = NULL; master->ats_enabled = false; - arm_smmu_install_ste_for_dev(master); - /* - * Clearing the CD entry isn't strictly required to detach the domain - * since the table is uninstalled anyway, but it helps avoid confusion - * in the call to arm_smmu_write_ctx_desc on the next attach (which - * expects the entry to be empty). - */ - if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 && master->cd_table.cdtab) - arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, NULL); } static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { int ret = 0; unsigned long flags; + struct arm_smmu_ste target; struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct arm_smmu_device *smmu; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); @@ -2398,15 +2524,10 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) return -EBUSY; } - arm_smmu_detach_dev(master); - mutex_lock(&smmu_domain->init_mutex); if (!smmu_domain->smmu) { - smmu_domain->smmu = smmu; - ret = arm_smmu_domain_finalise(domain); - if (ret) - smmu_domain->smmu = NULL; + ret = arm_smmu_domain_finalise(smmu_domain, smmu); } else if (smmu_domain->smmu != smmu) ret = -EINVAL; @@ -2414,57 +2535,140 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) if (ret) return ret; - master->domain = smmu_domain; - /* - * The SMMU does not support enabling ATS with bypass. When the STE is - * in bypass (STE.Config[2:0] == 0b100), ATS Translation Requests and - * Translated transactions are denied as though ATS is disabled for the - * stream (STE.EATS == 0b00), causing F_BAD_ATS_TREQ and - * F_TRANSL_FORBIDDEN events (IHI0070Ea 5.2 Stream Table Entry). + * Prevent arm_smmu_share_asid() from trying to change the ASID + * of either the old or new domain while we are working on it. + * This allows the STE and the smmu_domain->devices list to + * be inconsistent during this routine. */ - if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS) - master->ats_enabled = arm_smmu_ats_supported(master); + mutex_lock(&arm_smmu_asid_lock); + + arm_smmu_detach_dev(master); + + master->ats_enabled = arm_smmu_ats_supported(master); spin_lock_irqsave(&smmu_domain->devices_lock, flags); list_add(&master->domain_head, &smmu_domain->devices); spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); - if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { + switch (smmu_domain->stage) { + case ARM_SMMU_DOMAIN_S1: if (!master->cd_table.cdtab) { ret = arm_smmu_alloc_cd_tables(master); - if (ret) { - master->domain = NULL; + if (ret) + goto out_list_del; + } else { + /* + * arm_smmu_write_ctx_desc() relies on the entry being + * invalid to work, clear any existing entry. + */ + ret = arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, + NULL); + if (ret) goto out_list_del; - } } - /* - * Prevent SVA from concurrently modifying the CD or writing to - * the CD entry - */ - mutex_lock(&arm_smmu_asid_lock); ret = arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, &smmu_domain->cd); - mutex_unlock(&arm_smmu_asid_lock); - if (ret) { - master->domain = NULL; + if (ret) goto out_list_del; - } + + arm_smmu_make_cdtable_ste(&target, master); + arm_smmu_install_ste_for_dev(master, &target); + break; + case ARM_SMMU_DOMAIN_S2: + arm_smmu_make_s2_domain_ste(&target, master, smmu_domain); + arm_smmu_install_ste_for_dev(master, &target); + if (master->cd_table.cdtab) + arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, + NULL); + break; } - arm_smmu_install_ste_for_dev(master); - - arm_smmu_enable_ats(master); - return 0; + arm_smmu_enable_ats(master, smmu_domain); + goto out_unlock; out_list_del: spin_lock_irqsave(&smmu_domain->devices_lock, flags); - list_del(&master->domain_head); + list_del_init(&master->domain_head); spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); +out_unlock: + mutex_unlock(&arm_smmu_asid_lock); return ret; } +static int arm_smmu_attach_dev_ste(struct device *dev, + struct arm_smmu_ste *ste) +{ + struct arm_smmu_master *master = dev_iommu_priv_get(dev); + + if (arm_smmu_master_sva_enabled(master)) + return -EBUSY; + + /* + * Do not allow any ASID to be changed while are working on the STE, + * otherwise we could miss invalidations. + */ + mutex_lock(&arm_smmu_asid_lock); + + /* + * The SMMU does not support enabling ATS with bypass/abort. When the + * STE is in bypass (STE.Config[2:0] == 0b100), ATS Translation Requests + * and Translated transactions are denied as though ATS is disabled for + * the stream (STE.EATS == 0b00), causing F_BAD_ATS_TREQ and + * F_TRANSL_FORBIDDEN events (IHI0070Ea 5.2 Stream Table Entry). + */ + arm_smmu_detach_dev(master); + + arm_smmu_install_ste_for_dev(master, ste); + mutex_unlock(&arm_smmu_asid_lock); + + /* + * This has to be done after removing the master from the + * arm_smmu_domain->devices to avoid races updating the same context + * descriptor from arm_smmu_share_asid(). + */ + if (master->cd_table.cdtab) + arm_smmu_write_ctx_desc(master, IOMMU_NO_PASID, NULL); + return 0; +} + +static int arm_smmu_attach_dev_identity(struct iommu_domain *domain, + struct device *dev) +{ + struct arm_smmu_ste ste; + + arm_smmu_make_bypass_ste(&ste); + return arm_smmu_attach_dev_ste(dev, &ste); +} + +static const struct iommu_domain_ops arm_smmu_identity_ops = { + .attach_dev = arm_smmu_attach_dev_identity, +}; + +static struct iommu_domain arm_smmu_identity_domain = { + .type = IOMMU_DOMAIN_IDENTITY, + .ops = &arm_smmu_identity_ops, +}; + +static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain, + struct device *dev) +{ + struct arm_smmu_ste ste; + + arm_smmu_make_abort_ste(&ste); + return arm_smmu_attach_dev_ste(dev, &ste); +} + +static const struct iommu_domain_ops arm_smmu_blocked_ops = { + .attach_dev = arm_smmu_attach_dev_blocked, +}; + +static struct iommu_domain arm_smmu_blocked_domain = { + .type = IOMMU_DOMAIN_BLOCKED, + .ops = &arm_smmu_blocked_ops, +}; + static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t pgsize, size_t pgcount, int prot, gfp_t gfp, size_t *mapped) @@ -2658,6 +2862,7 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev) master->dev = dev; master->smmu = smmu; INIT_LIST_HEAD(&master->bonds); + INIT_LIST_HEAD(&master->domain_head); dev_iommu_priv_set(dev, master); ret = arm_smmu_insert_master(smmu, master); @@ -2699,7 +2904,13 @@ static void arm_smmu_release_device(struct device *dev) if (WARN_ON(arm_smmu_master_sva_enabled(master))) iopf_queue_remove_device(master->smmu->evtq.iopf, dev); - arm_smmu_detach_dev(master); + + /* Put the STE back to what arm_smmu_init_strtab() sets */ + if (disable_bypass && !dev->iommu->require_direct) + arm_smmu_attach_dev_blocked(&arm_smmu_blocked_domain, dev); + else + arm_smmu_attach_dev_identity(&arm_smmu_identity_domain, dev); + arm_smmu_disable_pasid(master); arm_smmu_remove_master(master); if (master->cd_table.cdtab) @@ -2739,7 +2950,8 @@ static int arm_smmu_enable_nesting(struct iommu_domain *domain) return ret; } -static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int arm_smmu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { return iommu_fwspec_add_ids(dev, args->args, 1); } @@ -2844,8 +3056,11 @@ static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid) } static struct iommu_ops arm_smmu_ops = { + .identity_domain = &arm_smmu_identity_domain, + .blocked_domain = &arm_smmu_blocked_domain, .capable = arm_smmu_capable, .domain_alloc = arm_smmu_domain_alloc, + .domain_alloc_paging = arm_smmu_domain_alloc_paging, .probe_device = arm_smmu_probe_device, .release_device = arm_smmu_release_device, .device_group = arm_smmu_device_group, @@ -3049,7 +3264,7 @@ static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits); cfg->strtab_base_cfg = reg; - arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents, false); + arm_smmu_init_initial_stes(strtab, cfg->num_l1_ents); return 0; } @@ -3761,7 +3976,6 @@ static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu) iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); list_for_each_entry(e, &rmr_list, list) { - struct arm_smmu_ste *step; struct iommu_iort_rmr_data *rmr; int ret, i; @@ -3774,8 +3988,12 @@ static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu) continue; } - step = arm_smmu_get_step_for_sid(smmu, rmr->sids[i]); - arm_smmu_init_bypass_stes(step, 1, true); + /* + * STE table is not programmed to HW, see + * arm_smmu_initial_bypass_stes() + */ + arm_smmu_make_bypass_ste( + arm_smmu_get_step_for_sid(smmu, rmr->sids[i])); } } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 65fb388d5173..23baf117e7e4 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -609,8 +609,6 @@ struct arm_smmu_ctx_desc_cfg { struct arm_smmu_s2_cfg { u16 vmid; - u64 vttbr; - u64 vtcr; }; struct arm_smmu_strtab_cfg { @@ -697,7 +695,6 @@ struct arm_smmu_stream { struct arm_smmu_master { struct arm_smmu_device *smmu; struct device *dev; - struct arm_smmu_domain *domain; struct list_head domain_head; struct arm_smmu_stream *streams; /* Locked by the iommu core using the group mutex */ @@ -715,7 +712,6 @@ struct arm_smmu_master { enum arm_smmu_domain_stage { ARM_SMMU_DOMAIN_S1 = 0, ARM_SMMU_DOMAIN_S2, - ARM_SMMU_DOMAIN_BYPASS, }; struct arm_smmu_domain { diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index 8b04ece00420..5c7cfc51b57c 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -260,6 +260,7 @@ static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = { { .compatible = "qcom,sm6375-mdss" }, { .compatible = "qcom,sm8150-mdss" }, { .compatible = "qcom,sm8250-mdss" }, + { .compatible = "qcom,x1e80100-mdss" }, { } }; diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index 6317aaf7b3ab..c572d877b0e1 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -1546,7 +1546,8 @@ static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain, return ret; } -static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int arm_smmu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { u32 mask, fwid = 0; diff --git a/drivers/iommu/arm/arm-smmu/qcom_iommu.c b/drivers/iommu/arm/arm-smmu/qcom_iommu.c index 17a1c163fef6..e079bb7a993e 100644 --- a/drivers/iommu/arm/arm-smmu/qcom_iommu.c +++ b/drivers/iommu/arm/arm-smmu/qcom_iommu.c @@ -546,7 +546,8 @@ static struct iommu_device *qcom_iommu_probe_device(struct device *dev) return &qcom_iommu->iommu; } -static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int qcom_iommu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { struct qcom_iommu_dev *qcom_iommu; struct platform_device *iommu_pdev; diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 4a90d8018202..55df94996f64 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -863,6 +863,11 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, iommu_deferred_attach(dev, domain)) return DMA_MAPPING_ERROR; + /* If anyone ever wants this we'd need support in the IOVA allocator */ + if (dev_WARN_ONCE(dev, dma_get_min_align_mask(dev) > iova_mask(iovad), + "Unsupported alignment constraint\n")) + return DMA_MAPPING_ERROR; + size = iova_align(iovad, size + iova_off); iova = iommu_dma_alloc_iova(domain, size, dma_mask, dev); diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 2c6e9094f1e9..d98c9161948a 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1431,7 +1431,7 @@ static void exynos_iommu_release_device(struct device *dev) } static int exynos_iommu_of_xlate(struct device *dev, - struct of_phandle_args *spec) + const struct of_phandle_args *spec) { struct platform_device *sysmmu = of_find_device_by_node(spec->np); struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev); diff --git a/drivers/iommu/intel/Kconfig b/drivers/iommu/intel/Kconfig index 012cd2541a68..6cf9f48e7d8c 100644 --- a/drivers/iommu/intel/Kconfig +++ b/drivers/iommu/intel/Kconfig @@ -51,6 +51,7 @@ config INTEL_IOMMU_SVM depends on X86_64 select MMU_NOTIFIER select IOMMU_SVA + select IOMMU_IOPF help Shared Virtual Memory (SVM) provides a facility for devices to access DMA resources through process address space by @@ -64,17 +65,6 @@ config INTEL_IOMMU_DEFAULT_ON one is found. If this option is not selected, DMAR support can be enabled by passing intel_iommu=on to the kernel. -config INTEL_IOMMU_BROKEN_GFX_WA - bool "Workaround broken graphics drivers (going away soon)" - depends on BROKEN && X86 - help - Current Graphics drivers tend to use physical address - for DMA and avoid using DMA APIs. Setting this config - option permits the IOMMU driver to set a unity map for - all the OS-visible memory. Hence the driver can continue - to use physical addresses for DMA, at least until this - option is removed in the 2.6.32 kernel. - config INTEL_IOMMU_FLOPPY_WA def_bool y depends on X86 diff --git a/drivers/iommu/intel/Makefile b/drivers/iommu/intel/Makefile index 5dabf081a779..5402b699a122 100644 --- a/drivers/iommu/intel/Makefile +++ b/drivers/iommu/intel/Makefile @@ -5,5 +5,7 @@ obj-$(CONFIG_DMAR_TABLE) += trace.o cap_audit.o obj-$(CONFIG_DMAR_PERF) += perf.o obj-$(CONFIG_INTEL_IOMMU_DEBUGFS) += debugfs.o obj-$(CONFIG_INTEL_IOMMU_SVM) += svm.o +ifdef CONFIG_INTEL_IOMMU obj-$(CONFIG_IRQ_REMAP) += irq_remapping.o +endif obj-$(CONFIG_INTEL_IOMMU_PERF_EVENTS) += perfmon.o diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 23cb80d62a9a..36d7427b1202 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1095,7 +1095,9 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd) iommu->agaw = agaw; iommu->msagaw = msagaw; iommu->segment = drhd->segment; - + iommu->device_rbtree = RB_ROOT; + spin_lock_init(&iommu->device_rbtree_lock); + mutex_init(&iommu->iopf_lock); iommu->node = NUMA_NO_NODE; ver = readl(iommu->reg + DMAR_VER_REG); @@ -1271,6 +1273,8 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) { u32 fault; int head, tail; + struct device *dev; + u64 iqe_err, ite_sid; struct q_inval *qi = iommu->qi; int shift = qi_shift(iommu); @@ -1315,6 +1319,13 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) tail = readl(iommu->reg + DMAR_IQT_REG); tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH; + /* + * SID field is valid only when the ITE field is Set in FSTS_REG + * see Intel VT-d spec r4.1, section 11.4.9.9 + */ + iqe_err = dmar_readq(iommu->reg + DMAR_IQER_REG); + ite_sid = DMAR_IQER_REG_ITESID(iqe_err); + writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG); pr_info("Invalidation Time-out Error (ITE) cleared\n"); @@ -1324,6 +1335,19 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) head = (head - 2 + QI_LENGTH) % QI_LENGTH; } while (head != tail); + /* + * If device was released or isn't present, no need to retry + * the ATS invalidate request anymore. + * + * 0 value of ite_sid means old VT-d device, no ite_sid value. + * see Intel VT-d spec r4.1, section 11.4.9.9 + */ + if (ite_sid) { + dev = device_rbtree_find(iommu, ite_sid); + if (!dev || !dev_is_pci(dev) || + !pci_device_is_present(to_pci_dev(dev))) + return -ETIMEDOUT; + } if (qi->desc_status[wait_index] == QI_ABORT) return -EAGAIN; } diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 11652e0bcab3..50eb9aed47cc 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -27,7 +27,6 @@ #include "iommu.h" #include "../dma-iommu.h" #include "../irq_remapping.h" -#include "../iommu-sva.h" #include "pasid.h" #include "cap_audit.h" #include "perfmon.h" @@ -97,6 +96,81 @@ static phys_addr_t root_entry_uctp(struct root_entry *re) return re->hi & VTD_PAGE_MASK; } +static int device_rid_cmp_key(const void *key, const struct rb_node *node) +{ + struct device_domain_info *info = + rb_entry(node, struct device_domain_info, node); + const u16 *rid_lhs = key; + + if (*rid_lhs < PCI_DEVID(info->bus, info->devfn)) + return -1; + + if (*rid_lhs > PCI_DEVID(info->bus, info->devfn)) + return 1; + + return 0; +} + +static int device_rid_cmp(struct rb_node *lhs, const struct rb_node *rhs) +{ + struct device_domain_info *info = + rb_entry(lhs, struct device_domain_info, node); + u16 key = PCI_DEVID(info->bus, info->devfn); + + return device_rid_cmp_key(&key, rhs); +} + +/* + * Looks up an IOMMU-probed device using its source ID. + * + * Returns the pointer to the device if there is a match. Otherwise, + * returns NULL. + * + * Note that this helper doesn't guarantee that the device won't be + * released by the iommu subsystem after being returned. The caller + * should use its own synchronization mechanism to avoid the device + * being released during its use if its possibly the case. + */ +struct device *device_rbtree_find(struct intel_iommu *iommu, u16 rid) +{ + struct device_domain_info *info = NULL; + struct rb_node *node; + unsigned long flags; + + spin_lock_irqsave(&iommu->device_rbtree_lock, flags); + node = rb_find(&rid, &iommu->device_rbtree, device_rid_cmp_key); + if (node) + info = rb_entry(node, struct device_domain_info, node); + spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags); + + return info ? info->dev : NULL; +} + +static int device_rbtree_insert(struct intel_iommu *iommu, + struct device_domain_info *info) +{ + struct rb_node *curr; + unsigned long flags; + + spin_lock_irqsave(&iommu->device_rbtree_lock, flags); + curr = rb_find_add(&info->node, &iommu->device_rbtree, device_rid_cmp); + spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags); + if (WARN_ON(curr)) + return -EEXIST; + + return 0; +} + +static void device_rbtree_remove(struct device_domain_info *info) +{ + struct intel_iommu *iommu = info->iommu; + unsigned long flags; + + spin_lock_irqsave(&iommu->device_rbtree_lock, flags); + rb_erase(&info->node, &iommu->device_rbtree); + spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags); +} + /* * This domain is a statically identity mapping domain. * 1. This domain creats a static 1:1 mapping to all usable memory. @@ -1776,34 +1850,17 @@ static void domain_exit(struct dmar_domain *domain) kfree(domain); } -/* - * Get the PASID directory size for scalable mode context entry. - * Value of X in the PDTS field of a scalable mode context entry - * indicates PASID directory with 2^(X + 7) entries. - */ -static unsigned long context_get_sm_pds(struct pasid_table *table) -{ - unsigned long pds, max_pde; - - max_pde = table->max_pasid >> PASID_PDE_SHIFT; - pds = find_first_bit(&max_pde, MAX_NR_PASID_BITS); - if (pds < 7) - return 0; - - return pds - 7; -} - static int domain_context_mapping_one(struct dmar_domain *domain, struct intel_iommu *iommu, - struct pasid_table *table, u8 bus, u8 devfn) { struct device_domain_info *info = domain_lookup_dev_info(domain, iommu, bus, devfn); u16 did = domain_id_iommu(domain, iommu); int translation = CONTEXT_TT_MULTI_LEVEL; + struct dma_pte *pgd = domain->pgd; struct context_entry *context; - int ret; + int agaw, ret; if (hw_pass_through && domain_type_is_si(domain)) translation = CONTEXT_TT_PASS_THROUGH; @@ -1846,65 +1903,37 @@ static int domain_context_mapping_one(struct dmar_domain *domain, } context_clear_entry(context); + context_set_domain_id(context, did); - if (sm_supported(iommu)) { - unsigned long pds; - - /* Setup the PASID DIR pointer: */ - pds = context_get_sm_pds(table); - context->lo = (u64)virt_to_phys(table->table) | - context_pdts(pds); - - /* Setup the RID_PASID field: */ - context_set_sm_rid2pasid(context, IOMMU_NO_PASID); - + if (translation != CONTEXT_TT_PASS_THROUGH) { /* - * Setup the Device-TLB enable bit and Page request - * Enable bit: + * Skip top levels of page tables for iommu which has + * less agaw than default. Unnecessary for PT mode. */ - if (info && info->ats_supported) - context_set_sm_dte(context); - if (info && info->pri_supported) - context_set_sm_pre(context); - if (info && info->pasid_supported) - context_set_pasid(context); - } else { - struct dma_pte *pgd = domain->pgd; - int agaw; - - context_set_domain_id(context, did); - - if (translation != CONTEXT_TT_PASS_THROUGH) { - /* - * Skip top levels of page tables for iommu which has - * less agaw than default. Unnecessary for PT mode. - */ - for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) { - ret = -ENOMEM; - pgd = phys_to_virt(dma_pte_addr(pgd)); - if (!dma_pte_present(pgd)) - goto out_unlock; - } - - if (info && info->ats_supported) - translation = CONTEXT_TT_DEV_IOTLB; - else - translation = CONTEXT_TT_MULTI_LEVEL; - - context_set_address_root(context, virt_to_phys(pgd)); - context_set_address_width(context, agaw); - } else { - /* - * In pass through mode, AW must be programmed to - * indicate the largest AGAW value supported by - * hardware. And ASR is ignored by hardware. - */ - context_set_address_width(context, iommu->msagaw); + for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) { + ret = -ENOMEM; + pgd = phys_to_virt(dma_pte_addr(pgd)); + if (!dma_pte_present(pgd)) + goto out_unlock; } - context_set_translation_type(context, translation); + if (info && info->ats_supported) + translation = CONTEXT_TT_DEV_IOTLB; + else + translation = CONTEXT_TT_MULTI_LEVEL; + + context_set_address_root(context, virt_to_phys(pgd)); + context_set_address_width(context, agaw); + } else { + /* + * In pass through mode, AW must be programmed to + * indicate the largest AGAW value supported by + * hardware. And ASR is ignored by hardware. + */ + context_set_address_width(context, iommu->msagaw); } + context_set_translation_type(context, translation); context_set_fault_enable(context); context_set_present(context); if (!ecap_coherent(iommu->ecap)) @@ -1934,43 +1963,29 @@ out_unlock: return ret; } -struct domain_context_mapping_data { - struct dmar_domain *domain; - struct intel_iommu *iommu; - struct pasid_table *table; -}; - static int domain_context_mapping_cb(struct pci_dev *pdev, u16 alias, void *opaque) { - struct domain_context_mapping_data *data = opaque; + struct device_domain_info *info = dev_iommu_priv_get(&pdev->dev); + struct intel_iommu *iommu = info->iommu; + struct dmar_domain *domain = opaque; - return domain_context_mapping_one(data->domain, data->iommu, - data->table, PCI_BUS_NUM(alias), - alias & 0xff); + return domain_context_mapping_one(domain, iommu, + PCI_BUS_NUM(alias), alias & 0xff); } static int domain_context_mapping(struct dmar_domain *domain, struct device *dev) { struct device_domain_info *info = dev_iommu_priv_get(dev); - struct domain_context_mapping_data data; struct intel_iommu *iommu = info->iommu; u8 bus = info->bus, devfn = info->devfn; - struct pasid_table *table; - - table = intel_pasid_get_table(dev); if (!dev_is_pci(dev)) - return domain_context_mapping_one(domain, iommu, table, - bus, devfn); - - data.domain = domain; - data.iommu = iommu; - data.table = table; + return domain_context_mapping_one(domain, iommu, bus, devfn); return pci_for_each_dma_alias(to_pci_dev(dev), - &domain_context_mapping_cb, &data); + domain_context_mapping_cb, domain); } /* Returns a number of VTD pages, but aligned to MM page size */ @@ -2160,9 +2175,6 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 struct context_entry *context; u16 did_old; - if (!iommu) - return; - spin_lock(&iommu->lock); context = iommu_context_addr(iommu, bus, devfn, 0); if (!context) { @@ -2170,14 +2182,7 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 return; } - if (sm_supported(iommu)) { - if (hw_pass_through && domain_type_is_si(info->domain)) - did_old = FLPT_DEFAULT_DID; - else - did_old = domain_id_iommu(info->domain, iommu); - } else { - did_old = context_domain_id(context); - } + did_old = context_domain_id(context); context_clear_entry(context); __iommu_flush_cache(iommu, context, sizeof(*context)); @@ -2188,9 +2193,6 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 DMA_CCMD_MASK_NOBIT, DMA_CCMD_DEVICE_INVL); - if (sm_supported(iommu)) - qi_flush_pasid_cache(iommu, did_old, QI_PC_ALL_PASIDS, 0); - iommu->flush.flush_iotlb(iommu, did_old, 0, @@ -2330,28 +2332,19 @@ static int dmar_domain_attach_device(struct dmar_domain *domain, list_add(&info->link, &domain->devices); spin_unlock_irqrestore(&domain->lock, flags); - /* PASID table is mandatory for a PCI device in scalable mode. */ - if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) { - /* Setup the PASID entry for requests without PASID: */ - if (hw_pass_through && domain_type_is_si(domain)) - ret = intel_pasid_setup_pass_through(iommu, - dev, IOMMU_NO_PASID); - else if (domain->use_first_level) - ret = domain_setup_first_level(iommu, domain, dev, - IOMMU_NO_PASID); - else - ret = intel_pasid_setup_second_level(iommu, domain, - dev, IOMMU_NO_PASID); - if (ret) { - dev_err(dev, "Setup RID2PASID failed\n"); - device_block_translation(dev); - return ret; - } - } + if (dev_is_real_dma_subdevice(dev)) + return 0; + + if (!sm_supported(iommu)) + ret = domain_context_mapping(domain, dev); + else if (hw_pass_through && domain_type_is_si(domain)) + ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID); + else if (domain->use_first_level) + ret = domain_setup_first_level(iommu, domain, dev, IOMMU_NO_PASID); + else + ret = intel_pasid_setup_second_level(iommu, domain, dev, IOMMU_NO_PASID); - ret = domain_context_mapping(domain, dev); if (ret) { - dev_err(dev, "Domain context map failed\n"); device_block_translation(dev); return ret; } @@ -2712,10 +2705,6 @@ static int __init init_dmars(void) iommu_set_root_entry(iommu); } -#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA - dmar_map_gfx = 0; -#endif - if (!dmar_map_gfx) iommu_identity_mapping |= IDENTMAP_GFX; @@ -3799,30 +3788,6 @@ static void domain_context_clear(struct device_domain_info *info) &domain_context_clear_one_cb, info); } -static void dmar_remove_one_dev_info(struct device *dev) -{ - struct device_domain_info *info = dev_iommu_priv_get(dev); - struct dmar_domain *domain = info->domain; - struct intel_iommu *iommu = info->iommu; - unsigned long flags; - - if (!dev_is_real_dma_subdevice(info->dev)) { - if (dev_is_pci(info->dev) && sm_supported(iommu)) - intel_pasid_tear_down_entry(iommu, info->dev, - IOMMU_NO_PASID, false); - - iommu_disable_pci_caps(info); - domain_context_clear(info); - } - - spin_lock_irqsave(&domain->lock, flags); - list_del(&info->link); - spin_unlock_irqrestore(&domain->lock, flags); - - domain_detach_iommu(domain, iommu); - info->domain = NULL; -} - /* * Clear the page table pointer in context or pasid table entries so that * all DMA requests without PASID from the device are blocked. If the page @@ -4027,6 +3992,10 @@ int prepare_domain_attach_device(struct iommu_domain *domain, dmar_domain->agaw--; } + if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev) && + context_copied(iommu, info->bus, info->devfn)) + return intel_pasid_setup_sm_context(dev); + return 0; } @@ -4330,26 +4299,50 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev) } dev_iommu_priv_set(dev, info); + ret = device_rbtree_insert(iommu, info); + if (ret) + goto free; if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) { ret = intel_pasid_alloc_table(dev); if (ret) { dev_err(dev, "PASID table allocation failed\n"); - kfree(info); - return ERR_PTR(ret); + goto clear_rbtree; + } + + if (!context_copied(iommu, info->bus, info->devfn)) { + ret = intel_pasid_setup_sm_context(dev); + if (ret) + goto free_table; } } intel_iommu_debugfs_create_dev(info); return &iommu->iommu; +free_table: + intel_pasid_free_table(dev); +clear_rbtree: + device_rbtree_remove(info); +free: + kfree(info); + + return ERR_PTR(ret); } static void intel_iommu_release_device(struct device *dev) { struct device_domain_info *info = dev_iommu_priv_get(dev); + struct intel_iommu *iommu = info->iommu; + + mutex_lock(&iommu->iopf_lock); + device_rbtree_remove(info); + mutex_unlock(&iommu->iopf_lock); + + if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev) && + !context_copied(iommu, info->bus, info->devfn)) + intel_pasid_teardown_sm_context(dev); - dmar_remove_one_dev_info(dev); intel_pasid_free_table(dev); intel_iommu_debugfs_remove_dev(info); kfree(info); @@ -4492,23 +4485,15 @@ static int intel_iommu_enable_iopf(struct device *dev) if (ret) return ret; - ret = iommu_register_device_fault_handler(dev, iommu_queue_iopf, dev); - if (ret) - goto iopf_remove_device; - ret = pci_enable_pri(pdev, PRQ_DEPTH); - if (ret) - goto iopf_unregister_handler; + if (ret) { + iopf_queue_remove_device(iommu->iopf_queue, dev); + return ret; + } + info->pri_enabled = 1; return 0; - -iopf_unregister_handler: - iommu_unregister_device_fault_handler(dev); -iopf_remove_device: - iopf_queue_remove_device(iommu->iopf_queue, dev); - - return ret; } static int intel_iommu_disable_iopf(struct device *dev) @@ -4529,14 +4514,7 @@ static int intel_iommu_disable_iopf(struct device *dev) */ pci_disable_pri(to_pci_dev(dev)); info->pri_enabled = 0; - - /* - * With PRI disabled and outstanding PRQs drained, unregistering - * fault handler and removing device from iopf queue should never - * fail. - */ - WARN_ON(iommu_unregister_device_fault_handler(dev)); - WARN_ON(iopf_queue_remove_device(iommu->iopf_queue, dev)); + iopf_queue_remove_device(iommu->iopf_queue, dev); return 0; } @@ -4855,6 +4833,7 @@ static const struct iommu_dirty_ops intel_dirty_ops = { const struct iommu_ops intel_iommu_ops = { .blocked_domain = &blocking_domain, + .release_domain = &blocking_domain, .capable = intel_iommu_capable, .hw_info = intel_iommu_hw_info, .domain_alloc = intel_iommu_domain_alloc, diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 4145c04cb1c6..404d2476a877 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -719,9 +719,16 @@ struct intel_iommu { #endif struct iopf_queue *iopf_queue; unsigned char iopfq_name[16]; + /* Synchronization between fault report and iommu device release. */ + struct mutex iopf_lock; struct q_inval *qi; /* Queued invalidation info */ u32 iommu_state[MAX_SR_DMAR_REGS]; /* Store iommu states between suspend and resume.*/ + /* rb tree for all probed devices */ + struct rb_root device_rbtree; + /* protect the device_rbtree */ + spinlock_t device_rbtree_lock; + #ifdef CONFIG_IRQ_REMAP struct ir_table *ir_table; /* Interrupt remapping info */ struct irq_domain *ir_domain; @@ -755,6 +762,8 @@ struct device_domain_info { struct intel_iommu *iommu; /* IOMMU used by this device */ struct dmar_domain *domain; /* pointer to domain */ struct pasid_table *pasid_table; /* pasid table */ + /* device tracking node(lookup by PCI RID) */ + struct rb_node node; #ifdef CONFIG_INTEL_IOMMU_DEBUGFS struct dentry *debugfs_dentry; /* pointer to device directory dentry */ #endif @@ -1081,13 +1090,14 @@ void free_pgtable_page(void *vaddr); void iommu_flush_write_buffer(struct intel_iommu *iommu); struct iommu_domain *intel_nested_domain_alloc(struct iommu_domain *parent, const struct iommu_user_data *user_data); +struct device *device_rbtree_find(struct intel_iommu *iommu, u16 rid); #ifdef CONFIG_INTEL_IOMMU_SVM void intel_svm_check(struct intel_iommu *iommu); int intel_svm_enable_prq(struct intel_iommu *iommu); int intel_svm_finish_prq(struct intel_iommu *iommu); -int intel_svm_page_response(struct device *dev, struct iommu_fault_event *evt, - struct iommu_page_response *msg); +void intel_svm_page_response(struct device *dev, struct iopf_fault *evt, + struct iommu_page_response *msg); struct iommu_domain *intel_svm_domain_alloc(void); void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid); void intel_drain_pasid_prq(struct device *dev, u32 pasid); diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 108158e2b907..11f0b856d74c 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -214,6 +214,9 @@ devtlb_invalidation_with_pasid(struct intel_iommu *iommu, if (!info || !info->ats_enabled) return; + if (pci_dev_is_disconnected(to_pci_dev(dev))) + return; + sid = info->bus << 8 | info->devfn; qdep = info->ats_qdep; pfsid = info->pfsid; @@ -667,3 +670,205 @@ int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev, return 0; } + +/* + * Interfaces to setup or teardown a pasid table to the scalable-mode + * context table entry: + */ + +static void device_pasid_table_teardown(struct device *dev, u8 bus, u8 devfn) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + struct intel_iommu *iommu = info->iommu; + struct context_entry *context; + + spin_lock(&iommu->lock); + context = iommu_context_addr(iommu, bus, devfn, false); + if (!context) { + spin_unlock(&iommu->lock); + return; + } + + context_clear_entry(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); + spin_unlock(&iommu->lock); + + /* + * Cache invalidation for changes to a scalable-mode context table + * entry. + * + * Section 6.5.3.3 of the VT-d spec: + * - Device-selective context-cache invalidation; + * - Domain-selective PASID-cache invalidation to affected domains + * (can be skipped if all PASID entries were not-present); + * - Domain-selective IOTLB invalidation to affected domains; + * - Global Device-TLB invalidation to affected functions. + * + * The iommu has been parked in the blocking state. All domains have + * been detached from the device or PASID. The PASID and IOTLB caches + * have been invalidated during the domain detach path. + */ + iommu->flush.flush_context(iommu, 0, PCI_DEVID(bus, devfn), + DMA_CCMD_MASK_NOBIT, DMA_CCMD_DEVICE_INVL); + devtlb_invalidation_with_pasid(iommu, dev, IOMMU_NO_PASID); +} + +static int pci_pasid_table_teardown(struct pci_dev *pdev, u16 alias, void *data) +{ + struct device *dev = data; + + if (dev == &pdev->dev) + device_pasid_table_teardown(dev, PCI_BUS_NUM(alias), alias & 0xff); + + return 0; +} + +void intel_pasid_teardown_sm_context(struct device *dev) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + + if (!dev_is_pci(dev)) { + device_pasid_table_teardown(dev, info->bus, info->devfn); + return; + } + + pci_for_each_dma_alias(to_pci_dev(dev), pci_pasid_table_teardown, dev); +} + +/* + * Get the PASID directory size for scalable mode context entry. + * Value of X in the PDTS field of a scalable mode context entry + * indicates PASID directory with 2^(X + 7) entries. + */ +static unsigned long context_get_sm_pds(struct pasid_table *table) +{ + unsigned long pds, max_pde; + + max_pde = table->max_pasid >> PASID_PDE_SHIFT; + pds = find_first_bit(&max_pde, MAX_NR_PASID_BITS); + if (pds < 7) + return 0; + + return pds - 7; +} + +static int context_entry_set_pasid_table(struct context_entry *context, + struct device *dev) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + struct pasid_table *table = info->pasid_table; + struct intel_iommu *iommu = info->iommu; + unsigned long pds; + + context_clear_entry(context); + + pds = context_get_sm_pds(table); + context->lo = (u64)virt_to_phys(table->table) | context_pdts(pds); + context_set_sm_rid2pasid(context, IOMMU_NO_PASID); + + if (info->ats_supported) + context_set_sm_dte(context); + if (info->pri_supported) + context_set_sm_pre(context); + if (info->pasid_supported) + context_set_pasid(context); + + context_set_fault_enable(context); + context_set_present(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); + + return 0; +} + +static int device_pasid_table_setup(struct device *dev, u8 bus, u8 devfn) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + struct intel_iommu *iommu = info->iommu; + struct context_entry *context; + + spin_lock(&iommu->lock); + context = iommu_context_addr(iommu, bus, devfn, true); + if (!context) { + spin_unlock(&iommu->lock); + return -ENOMEM; + } + + if (context_present(context) && !context_copied(iommu, bus, devfn)) { + spin_unlock(&iommu->lock); + return 0; + } + + if (context_copied(iommu, bus, devfn)) { + context_clear_entry(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); + + /* + * For kdump cases, old valid entries may be cached due to + * the in-flight DMA and copied pgtable, but there is no + * unmapping behaviour for them, thus we need explicit cache + * flushes for all affected domain IDs and PASIDs used in + * the copied PASID table. Given that we have no idea about + * which domain IDs and PASIDs were used in the copied tables, + * upgrade them to global PASID and IOTLB cache invalidation. + */ + iommu->flush.flush_context(iommu, 0, + PCI_DEVID(bus, devfn), + DMA_CCMD_MASK_NOBIT, + DMA_CCMD_DEVICE_INVL); + qi_flush_pasid_cache(iommu, 0, QI_PC_GLOBAL, 0); + iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); + devtlb_invalidation_with_pasid(iommu, dev, IOMMU_NO_PASID); + + /* + * At this point, the device is supposed to finish reset at + * its driver probe stage, so no in-flight DMA will exist, + * and we don't need to worry anymore hereafter. + */ + clear_context_copied(iommu, bus, devfn); + } + + context_entry_set_pasid_table(context, dev); + spin_unlock(&iommu->lock); + + /* + * It's a non-present to present mapping. If hardware doesn't cache + * non-present entry we don't need to flush the caches. If it does + * cache non-present entries, then it does so in the special + * domain #0, which we have to flush: + */ + if (cap_caching_mode(iommu->cap)) { + iommu->flush.flush_context(iommu, 0, + PCI_DEVID(bus, devfn), + DMA_CCMD_MASK_NOBIT, + DMA_CCMD_DEVICE_INVL); + iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH); + } + + return 0; +} + +static int pci_pasid_table_setup(struct pci_dev *pdev, u16 alias, void *data) +{ + struct device *dev = data; + + if (dev != &pdev->dev) + return 0; + + return device_pasid_table_setup(dev, PCI_BUS_NUM(alias), alias & 0xff); +} + +/* + * Set the device's PASID table to its context table entry. + * + * The PASID table is set to the context entries of both device itself + * and its alias requester ID for DMA. + */ +int intel_pasid_setup_sm_context(struct device *dev) +{ + struct device_domain_info *info = dev_iommu_priv_get(dev); + + if (!dev_is_pci(dev)) + return device_pasid_table_setup(dev, info->bus, info->devfn); + + return pci_for_each_dma_alias(to_pci_dev(dev), pci_pasid_table_setup, dev); +} diff --git a/drivers/iommu/intel/pasid.h b/drivers/iommu/intel/pasid.h index 487ede039bdd..da9978fef7ac 100644 --- a/drivers/iommu/intel/pasid.h +++ b/drivers/iommu/intel/pasid.h @@ -318,4 +318,6 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, bool fault_ignore); void intel_pasid_setup_page_snoop_control(struct intel_iommu *iommu, struct device *dev, u32 pasid); +int intel_pasid_setup_sm_context(struct device *dev); +void intel_pasid_teardown_sm_context(struct device *dev); #endif /* __INTEL_PASID_H */ diff --git a/drivers/iommu/intel/perf.c b/drivers/iommu/intel/perf.c index 94ee70ac38e3..adc4de6bbd88 100644 --- a/drivers/iommu/intel/perf.c +++ b/drivers/iommu/intel/perf.c @@ -33,7 +33,7 @@ int dmar_latency_enable(struct intel_iommu *iommu, enum latency_type type) spin_lock_irqsave(&latency_lock, flags); if (!iommu->perf_statistic) { - iommu->perf_statistic = kzalloc(sizeof(*lstat) * DMAR_LATENCY_NUM, + iommu->perf_statistic = kcalloc(DMAR_LATENCY_NUM, sizeof(*lstat), GFP_ATOMIC); if (!iommu->perf_statistic) { ret = -ENOMEM; diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 40edd282903f..c1bed89b1026 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -22,7 +22,6 @@ #include "iommu.h" #include "pasid.h" #include "perf.h" -#include "../iommu-sva.h" #include "trace.h" static irqreturn_t prq_event_thread(int irq, void *d); @@ -315,10 +314,11 @@ out: return 0; } -static int intel_svm_bind_mm(struct intel_iommu *iommu, struct device *dev, - struct iommu_domain *domain, ioasid_t pasid) +static int intel_svm_set_dev_pasid(struct iommu_domain *domain, + struct device *dev, ioasid_t pasid) { struct device_domain_info *info = dev_iommu_priv_get(dev); + struct intel_iommu *iommu = info->iommu; struct mm_struct *mm = domain->mm; struct intel_svm_dev *sdev; struct intel_svm *svm; @@ -360,7 +360,6 @@ static int intel_svm_bind_mm(struct intel_iommu *iommu, struct device *dev, sdev->iommu = iommu; sdev->did = FLPT_DEFAULT_DID; sdev->sid = PCI_DEVID(info->bus, info->devfn); - init_rcu_head(&sdev->rcu); if (info->ats_enabled) { sdev->qdep = info->ats_qdep; if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS) @@ -408,13 +407,6 @@ void intel_svm_remove_dev_pasid(struct device *dev, u32 pasid) if (svm->notifier.ops) mmu_notifier_unregister(&svm->notifier, mm); pasid_private_remove(svm->pasid); - /* - * We mandate that no page faults may be outstanding - * for the PASID when intel_svm_unbind_mm() is called. - * If that is not obeyed, subtle errors will happen. - * Let's make them less subtle... - */ - memset(svm, 0x6b, sizeof(*svm)); kfree(svm); } } @@ -562,16 +554,12 @@ static int prq_to_iommu_prot(struct page_req_dsc *req) return prot; } -static int intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, - struct page_req_dsc *desc) +static void intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, + struct page_req_dsc *desc) { - struct iommu_fault_event event; - - if (!dev || !dev_is_pci(dev)) - return -ENODEV; + struct iopf_fault event = { }; /* Fill in event data for device specific processing */ - memset(&event, 0, sizeof(struct iommu_fault_event)); event.fault.type = IOMMU_FAULT_PAGE_REQ; event.fault.prm.addr = (u64)desc->addr << VTD_PAGE_SHIFT; event.fault.prm.pasid = desc->pasid; @@ -603,7 +591,7 @@ static int intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, event.fault.prm.private_data[0] = ktime_to_ns(ktime_get()); } - return iommu_report_device_fault(dev, &event); + iommu_report_device_fault(dev, &event); } static void handle_bad_prq_event(struct intel_iommu *iommu, @@ -650,7 +638,7 @@ static irqreturn_t prq_event_thread(int irq, void *d) struct intel_iommu *iommu = d; struct page_req_dsc *req; int head, tail, handled; - struct pci_dev *pdev; + struct device *dev; u64 address; /* @@ -696,23 +684,22 @@ bad_req: if (unlikely(req->lpig && !req->rd_req && !req->wr_req)) goto prq_advance; - pdev = pci_get_domain_bus_and_slot(iommu->segment, - PCI_BUS_NUM(req->rid), - req->rid & 0xff); /* * If prq is to be handled outside iommu driver via receiver of * the fault notifiers, we skip the page response here. */ - if (!pdev) + mutex_lock(&iommu->iopf_lock); + dev = device_rbtree_find(iommu, req->rid); + if (!dev) { + mutex_unlock(&iommu->iopf_lock); goto bad_req; + } - if (intel_svm_prq_report(iommu, &pdev->dev, req)) - handle_bad_prq_event(iommu, req, QI_RESP_INVALID); - else - trace_prq_report(iommu, &pdev->dev, req->qw_0, req->qw_1, - req->priv_data[0], req->priv_data[1], - iommu->prq_seq_number++); - pci_dev_put(pdev); + intel_svm_prq_report(iommu, dev, req); + trace_prq_report(iommu, dev, req->qw_0, req->qw_1, + req->priv_data[0], req->priv_data[1], + iommu->prq_seq_number++); + mutex_unlock(&iommu->iopf_lock); prq_advance: head = (head + sizeof(*req)) & PRQ_RING_MASK; } @@ -742,9 +729,8 @@ prq_advance: return IRQ_RETVAL(handled); } -int intel_svm_page_response(struct device *dev, - struct iommu_fault_event *evt, - struct iommu_page_response *msg) +void intel_svm_page_response(struct device *dev, struct iopf_fault *evt, + struct iommu_page_response *msg) { struct device_domain_info *info = dev_iommu_priv_get(dev); struct intel_iommu *iommu = info->iommu; @@ -753,7 +739,6 @@ int intel_svm_page_response(struct device *dev, bool private_present; bool pasid_present; bool last_page; - int ret = 0; u16 sid; prm = &evt->fault.prm; @@ -762,16 +747,6 @@ int intel_svm_page_response(struct device *dev, private_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; last_page = prm->flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; - if (!pasid_present) { - ret = -EINVAL; - goto out; - } - - if (prm->pasid == 0 || prm->pasid >= PASID_MAX) { - ret = -EINVAL; - goto out; - } - /* * Per VT-d spec. v3.0 ch7.7, system software must respond * with page group response if private data is present (PDP) @@ -800,17 +775,6 @@ int intel_svm_page_response(struct device *dev, qi_submit_sync(iommu, &desc, 1, 0); } -out: - return ret; -} - -static int intel_svm_set_dev_pasid(struct iommu_domain *domain, - struct device *dev, ioasid_t pasid) -{ - struct device_domain_info *info = dev_iommu_priv_get(dev); - struct intel_iommu *iommu = info->iommu; - - return intel_svm_bind_mm(iommu, dev, domain, pasid); } static void intel_svm_domain_free(struct iommu_domain *domain) diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index e5b8b9110c13..06d78fcc79fd 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -11,101 +11,140 @@ #include #include -#include "iommu-sva.h" +#include "iommu-priv.h" -/** - * struct iopf_queue - IO Page Fault queue - * @wq: the fault workqueue - * @devices: devices attached to this queue - * @lock: protects the device list +/* + * Return the fault parameter of a device if it exists. Otherwise, return NULL. + * On a successful return, the caller takes a reference of this parameter and + * should put it after use by calling iopf_put_dev_fault_param(). */ -struct iopf_queue { - struct workqueue_struct *wq; - struct list_head devices; - struct mutex lock; -}; - -/** - * struct iopf_device_param - IO Page Fault data attached to a device - * @dev: the device that owns this param - * @queue: IOPF queue - * @queue_list: index into queue->devices - * @partial: faults that are part of a Page Request Group for which the last - * request hasn't been submitted yet. - */ -struct iopf_device_param { - struct device *dev; - struct iopf_queue *queue; - struct list_head queue_list; - struct list_head partial; -}; - -struct iopf_fault { - struct iommu_fault fault; - struct list_head list; -}; - -struct iopf_group { - struct iopf_fault last_fault; - struct list_head faults; - struct work_struct work; - struct device *dev; -}; - -static int iopf_complete_group(struct device *dev, struct iopf_fault *iopf, - enum iommu_page_response_code status) +static struct iommu_fault_param *iopf_get_dev_fault_param(struct device *dev) { - struct iommu_page_response resp = { - .version = IOMMU_PAGE_RESP_VERSION_1, - .pasid = iopf->fault.prm.pasid, - .grpid = iopf->fault.prm.grpid, - .code = status, - }; + struct dev_iommu *param = dev->iommu; + struct iommu_fault_param *fault_param; - if ((iopf->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) && - (iopf->fault.prm.flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID)) - resp.flags = IOMMU_PAGE_RESP_PASID_VALID; + rcu_read_lock(); + fault_param = rcu_dereference(param->fault_param); + if (fault_param && !refcount_inc_not_zero(&fault_param->users)) + fault_param = NULL; + rcu_read_unlock(); - return iommu_page_response(dev, &resp); + return fault_param; } -static void iopf_handler(struct work_struct *work) +/* Caller must hold a reference of the fault parameter. */ +static void iopf_put_dev_fault_param(struct iommu_fault_param *fault_param) { - struct iopf_group *group; - struct iommu_domain *domain; - struct iopf_fault *iopf, *next; - enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS; + if (refcount_dec_and_test(&fault_param->users)) + kfree_rcu(fault_param, rcu); +} - group = container_of(work, struct iopf_group, work); - domain = iommu_get_domain_for_dev_pasid(group->dev, - group->last_fault.fault.prm.pasid, 0); - if (!domain || !domain->iopf_handler) - status = IOMMU_PAGE_RESP_INVALID; +static void __iopf_free_group(struct iopf_group *group) +{ + struct iopf_fault *iopf, *next; list_for_each_entry_safe(iopf, next, &group->faults, list) { - /* - * For the moment, errors are sticky: don't handle subsequent - * faults in the group if there is an error. - */ - if (status == IOMMU_PAGE_RESP_SUCCESS) - status = domain->iopf_handler(&iopf->fault, - domain->fault_data); - - if (!(iopf->fault.prm.flags & - IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) + if (!(iopf->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) kfree(iopf); } - iopf_complete_group(group->dev, &group->last_fault, status); + /* Pair with iommu_report_device_fault(). */ + iopf_put_dev_fault_param(group->fault_param); +} + +void iopf_free_group(struct iopf_group *group) +{ + __iopf_free_group(group); kfree(group); } +EXPORT_SYMBOL_GPL(iopf_free_group); + +static struct iommu_domain *get_domain_for_iopf(struct device *dev, + struct iommu_fault *fault) +{ + struct iommu_domain *domain; + + if (fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) { + domain = iommu_get_domain_for_dev_pasid(dev, fault->prm.pasid, 0); + if (IS_ERR(domain)) + domain = NULL; + } else { + domain = iommu_get_domain_for_dev(dev); + } + + if (!domain || !domain->iopf_handler) { + dev_warn_ratelimited(dev, + "iopf (pasid %d) without domain attached or handler installed\n", + fault->prm.pasid); + + return NULL; + } + + return domain; +} + +/* Non-last request of a group. Postpone until the last one. */ +static int report_partial_fault(struct iommu_fault_param *fault_param, + struct iommu_fault *fault) +{ + struct iopf_fault *iopf; + + iopf = kzalloc(sizeof(*iopf), GFP_KERNEL); + if (!iopf) + return -ENOMEM; + + iopf->fault = *fault; + + mutex_lock(&fault_param->lock); + list_add(&iopf->list, &fault_param->partial); + mutex_unlock(&fault_param->lock); + + return 0; +} + +static struct iopf_group *iopf_group_alloc(struct iommu_fault_param *iopf_param, + struct iopf_fault *evt, + struct iopf_group *abort_group) +{ + struct iopf_fault *iopf, *next; + struct iopf_group *group; + + group = kzalloc(sizeof(*group), GFP_KERNEL); + if (!group) { + /* + * We always need to construct the group as we need it to abort + * the request at the driver if it can't be handled. + */ + group = abort_group; + } + + group->fault_param = iopf_param; + group->last_fault.fault = evt->fault; + INIT_LIST_HEAD(&group->faults); + INIT_LIST_HEAD(&group->pending_node); + list_add(&group->last_fault.list, &group->faults); + + /* See if we have partial faults for this group */ + mutex_lock(&iopf_param->lock); + list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { + if (iopf->fault.prm.grpid == evt->fault.prm.grpid) + /* Insert *before* the last fault */ + list_move(&iopf->list, &group->faults); + } + list_add(&group->pending_node, &iopf_param->faults); + mutex_unlock(&iopf_param->lock); + + return group; +} /** - * iommu_queue_iopf - IO Page Fault handler - * @fault: fault event - * @cookie: struct device, passed to iommu_register_device_fault_handler. + * iommu_report_device_fault() - Report fault event to device driver + * @dev: the device + * @evt: fault event data * - * Add a fault to the device workqueue, to be handled by mm. + * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ + * handler. If this function fails then ops->page_response() was called to + * complete evt if required. * * This module doesn't handle PCI PASID Stop Marker; IOMMU drivers must discard * them before reporting faults. A PASID Stop Marker (LRW = 0b100) doesn't @@ -137,83 +176,57 @@ static void iopf_handler(struct work_struct *work) * freed after the device has stopped generating page faults (or the iommu * hardware has been set to block the page faults) and the pending page faults * have been flushed. - * - * Return: 0 on success and <0 on error. */ -int iommu_queue_iopf(struct iommu_fault *fault, void *cookie) +void iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) { - int ret; + struct iommu_fault *fault = &evt->fault; + struct iommu_fault_param *iopf_param; + struct iopf_group abort_group = {}; struct iopf_group *group; - struct iopf_fault *iopf, *next; - struct iopf_device_param *iopf_param; - struct device *dev = cookie; - struct dev_iommu *param = dev->iommu; - - lockdep_assert_held(¶m->lock); - - if (fault->type != IOMMU_FAULT_PAGE_REQ) - /* Not a recoverable page fault */ - return -EOPNOTSUPP; - - /* - * As long as we're holding param->lock, the queue can't be unlinked - * from the device and therefore cannot disappear. - */ - iopf_param = param->iopf_param; - if (!iopf_param) - return -ENODEV; + iopf_param = iopf_get_dev_fault_param(dev); + if (WARN_ON(!iopf_param)) + return; if (!(fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { - iopf = kzalloc(sizeof(*iopf), GFP_KERNEL); - if (!iopf) - return -ENOMEM; - - iopf->fault = *fault; - - /* Non-last request of a group. Postpone until the last one */ - list_add(&iopf->list, &iopf_param->partial); - - return 0; + report_partial_fault(iopf_param, fault); + iopf_put_dev_fault_param(iopf_param); + /* A request that is not the last does not need to be ack'd */ } - group = kzalloc(sizeof(*group), GFP_KERNEL); - if (!group) { - /* - * The caller will send a response to the hardware. But we do - * need to clean up before leaving, otherwise partial faults - * will be stuck. - */ - ret = -ENOMEM; - goto cleanup_partial; - } + /* + * This is the last page fault of a group. Allocate an iopf group and + * pass it to domain's page fault handler. The group holds a reference + * count of the fault parameter. It will be released after response or + * error path of this function. If an error is returned, the caller + * will send a response to the hardware. We need to clean up before + * leaving, otherwise partial faults will be stuck. + */ + group = iopf_group_alloc(iopf_param, evt, &abort_group); + if (group == &abort_group) + goto err_abort; - group->dev = dev; - group->last_fault.fault = *fault; - INIT_LIST_HEAD(&group->faults); - list_add(&group->last_fault.list, &group->faults); - INIT_WORK(&group->work, iopf_handler); + group->domain = get_domain_for_iopf(dev, fault); + if (!group->domain) + goto err_abort; - /* See if we have partial faults for this group */ - list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { - if (iopf->fault.prm.grpid == fault->prm.grpid) - /* Insert *before* the last fault */ - list_move(&iopf->list, &group->faults); - } + /* + * On success iopf_handler must call iopf_group_response() and + * iopf_free_group() + */ + if (group->domain->iopf_handler(group)) + goto err_abort; - queue_work(iopf_param->queue->wq, &group->work); - return 0; + return; -cleanup_partial: - list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { - if (iopf->fault.prm.grpid == fault->prm.grpid) { - list_del(&iopf->list); - kfree(iopf); - } - } - return ret; +err_abort: + iopf_group_response(group, IOMMU_PAGE_RESP_FAILURE); + if (group == &abort_group) + __iopf_free_group(group); + else + iopf_free_group(group); } -EXPORT_SYMBOL_GPL(iommu_queue_iopf); +EXPORT_SYMBOL_GPL(iommu_report_device_fault); /** * iopf_queue_flush_dev - Ensure that all queued faults have been processed @@ -229,25 +242,51 @@ EXPORT_SYMBOL_GPL(iommu_queue_iopf); */ int iopf_queue_flush_dev(struct device *dev) { - int ret = 0; - struct iopf_device_param *iopf_param; - struct dev_iommu *param = dev->iommu; + struct iommu_fault_param *iopf_param; - if (!param) + /* + * It's a driver bug to be here after iopf_queue_remove_device(). + * Therefore, it's safe to dereference the fault parameter without + * holding the lock. + */ + iopf_param = rcu_dereference_check(dev->iommu->fault_param, true); + if (WARN_ON(!iopf_param)) return -ENODEV; - mutex_lock(¶m->lock); - iopf_param = param->iopf_param; - if (iopf_param) - flush_workqueue(iopf_param->queue->wq); - else - ret = -ENODEV; - mutex_unlock(¶m->lock); + flush_workqueue(iopf_param->queue->wq); - return ret; + return 0; } EXPORT_SYMBOL_GPL(iopf_queue_flush_dev); +/** + * iopf_group_response - Respond a group of page faults + * @group: the group of faults with the same group id + * @status: the response code + */ +void iopf_group_response(struct iopf_group *group, + enum iommu_page_response_code status) +{ + struct iommu_fault_param *fault_param = group->fault_param; + struct iopf_fault *iopf = &group->last_fault; + struct device *dev = group->fault_param->dev; + const struct iommu_ops *ops = dev_iommu_ops(dev); + struct iommu_page_response resp = { + .pasid = iopf->fault.prm.pasid, + .grpid = iopf->fault.prm.grpid, + .code = status, + }; + + /* Only send response if there is a fault report pending */ + mutex_lock(&fault_param->lock); + if (!list_empty(&group->pending_node)) { + ops->page_response(dev, &group->last_fault, &resp); + list_del_init(&group->pending_node); + } + mutex_unlock(&fault_param->lock); +} +EXPORT_SYMBOL_GPL(iopf_group_response); + /** * iopf_queue_discard_partial - Remove all pending partial fault * @queue: the queue whose partial faults need to be discarded @@ -261,18 +300,20 @@ EXPORT_SYMBOL_GPL(iopf_queue_flush_dev); int iopf_queue_discard_partial(struct iopf_queue *queue) { struct iopf_fault *iopf, *next; - struct iopf_device_param *iopf_param; + struct iommu_fault_param *iopf_param; if (!queue) return -EINVAL; mutex_lock(&queue->lock); list_for_each_entry(iopf_param, &queue->devices, queue_list) { + mutex_lock(&iopf_param->lock); list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) { list_del(&iopf->list); kfree(iopf); } + mutex_unlock(&iopf_param->lock); } mutex_unlock(&queue->lock); return 0; @@ -288,34 +329,42 @@ EXPORT_SYMBOL_GPL(iopf_queue_discard_partial); */ int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev) { - int ret = -EBUSY; - struct iopf_device_param *iopf_param; + int ret = 0; struct dev_iommu *param = dev->iommu; + struct iommu_fault_param *fault_param; + const struct iommu_ops *ops = dev_iommu_ops(dev); - if (!param) + if (!ops->page_response) return -ENODEV; - iopf_param = kzalloc(sizeof(*iopf_param), GFP_KERNEL); - if (!iopf_param) - return -ENOMEM; - - INIT_LIST_HEAD(&iopf_param->partial); - iopf_param->queue = queue; - iopf_param->dev = dev; - mutex_lock(&queue->lock); mutex_lock(¶m->lock); - if (!param->iopf_param) { - list_add(&iopf_param->queue_list, &queue->devices); - param->iopf_param = iopf_param; - ret = 0; + if (rcu_dereference_check(param->fault_param, + lockdep_is_held(¶m->lock))) { + ret = -EBUSY; + goto done_unlock; } + + fault_param = kzalloc(sizeof(*fault_param), GFP_KERNEL); + if (!fault_param) { + ret = -ENOMEM; + goto done_unlock; + } + + mutex_init(&fault_param->lock); + INIT_LIST_HEAD(&fault_param->faults); + INIT_LIST_HEAD(&fault_param->partial); + fault_param->dev = dev; + refcount_set(&fault_param->users, 1); + list_add(&fault_param->queue_list, &queue->devices); + fault_param->queue = queue; + + rcu_assign_pointer(param->fault_param, fault_param); + +done_unlock: mutex_unlock(¶m->lock); mutex_unlock(&queue->lock); - if (ret) - kfree(iopf_param); - return ret; } EXPORT_SYMBOL_GPL(iopf_queue_add_device); @@ -325,40 +374,66 @@ EXPORT_SYMBOL_GPL(iopf_queue_add_device); * @queue: IOPF queue * @dev: device to remove * - * Caller makes sure that no more faults are reported for this device. + * Removing a device from an iopf_queue. It's recommended to follow these + * steps when removing a device: * - * Return: 0 on success and <0 on error. + * - Disable new PRI reception: Turn off PRI generation in the IOMMU hardware + * and flush any hardware page request queues. This should be done before + * calling into this helper. + * - Acknowledge all outstanding PRQs to the device: Respond to all outstanding + * page requests with IOMMU_PAGE_RESP_INVALID, indicating the device should + * not retry. This helper function handles this. + * - Disable PRI on the device: After calling this helper, the caller could + * then disable PRI on the device. + * + * Calling iopf_queue_remove_device() essentially disassociates the device. + * The fault_param might still exist, but iommu_page_response() will do + * nothing. The device fault parameter reference count has been properly + * passed from iommu_report_device_fault() to the fault handling work, and + * will eventually be released after iommu_page_response(). */ -int iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) +void iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) { - int ret = -EINVAL; - struct iopf_fault *iopf, *next; - struct iopf_device_param *iopf_param; + struct iopf_fault *partial_iopf; + struct iopf_fault *next; + struct iopf_group *group, *temp; struct dev_iommu *param = dev->iommu; - - if (!param || !queue) - return -EINVAL; + struct iommu_fault_param *fault_param; + const struct iommu_ops *ops = dev_iommu_ops(dev); mutex_lock(&queue->lock); mutex_lock(¶m->lock); - iopf_param = param->iopf_param; - if (iopf_param && iopf_param->queue == queue) { - list_del(&iopf_param->queue_list); - param->iopf_param = NULL; - ret = 0; + fault_param = rcu_dereference_check(param->fault_param, + lockdep_is_held(¶m->lock)); + + if (WARN_ON(!fault_param || fault_param->queue != queue)) + goto unlock; + + mutex_lock(&fault_param->lock); + list_for_each_entry_safe(partial_iopf, next, &fault_param->partial, list) + kfree(partial_iopf); + + list_for_each_entry_safe(group, temp, &fault_param->faults, pending_node) { + struct iopf_fault *iopf = &group->last_fault; + struct iommu_page_response resp = { + .pasid = iopf->fault.prm.pasid, + .grpid = iopf->fault.prm.grpid, + .code = IOMMU_PAGE_RESP_INVALID + }; + + ops->page_response(dev, iopf, &resp); + list_del_init(&group->pending_node); } + mutex_unlock(&fault_param->lock); + + list_del(&fault_param->queue_list); + + /* dec the ref owned by iopf_queue_add_device() */ + rcu_assign_pointer(param->fault_param, NULL); + iopf_put_dev_fault_param(fault_param); +unlock: mutex_unlock(¶m->lock); mutex_unlock(&queue->lock); - if (ret) - return ret; - - /* Just in case some faults are still stuck */ - list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) - kfree(iopf); - - kfree(iopf_param); - - return 0; } EXPORT_SYMBOL_GPL(iopf_queue_remove_device); @@ -404,7 +479,7 @@ EXPORT_SYMBOL_GPL(iopf_queue_alloc); */ void iopf_queue_free(struct iopf_queue *queue) { - struct iopf_device_param *iopf_param, *next; + struct iommu_fault_param *iopf_param, *next; if (!queue) return; diff --git a/drivers/iommu/iommu-priv.h b/drivers/iommu/iommu-priv.h index 2024a2313348..5f731d994803 100644 --- a/drivers/iommu/iommu-priv.h +++ b/drivers/iommu/iommu-priv.h @@ -21,10 +21,11 @@ int iommu_group_replace_domain(struct iommu_group *group, struct iommu_domain *new_domain); int iommu_device_register_bus(struct iommu_device *iommu, - const struct iommu_ops *ops, struct bus_type *bus, + const struct iommu_ops *ops, + const struct bus_type *bus, struct notifier_block *nb); void iommu_device_unregister_bus(struct iommu_device *iommu, - struct bus_type *bus, + const struct bus_type *bus, struct notifier_block *nb); #endif /* __LINUX_IOMMU_PRIV_H */ diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index 65814cbc8402..640acc804e8c 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -7,7 +7,7 @@ #include #include -#include "iommu-sva.h" +#include "iommu-priv.h" static DEFINE_MUTEX(iommu_sva_lock); @@ -176,15 +176,25 @@ u32 iommu_sva_get_pasid(struct iommu_sva *handle) } EXPORT_SYMBOL_GPL(iommu_sva_get_pasid); +void mm_pasid_drop(struct mm_struct *mm) +{ + struct iommu_mm_data *iommu_mm = mm->iommu_mm; + + if (!iommu_mm) + return; + + iommu_free_global_pasid(iommu_mm->pasid); + kfree(iommu_mm); +} + /* * I/O page fault handler for SVA */ -enum iommu_page_response_code -iommu_sva_handle_iopf(struct iommu_fault *fault, void *data) +static enum iommu_page_response_code +iommu_sva_handle_mm(struct iommu_fault *fault, struct mm_struct *mm) { vm_fault_t ret; struct vm_area_struct *vma; - struct mm_struct *mm = data; unsigned int access_flags = 0; unsigned int fault_flags = FAULT_FLAG_REMOTE; struct iommu_fault_page_request *prm = &fault->prm; @@ -234,13 +244,54 @@ out_put_mm: return status; } -void mm_pasid_drop(struct mm_struct *mm) +static void iommu_sva_handle_iopf(struct work_struct *work) { - struct iommu_mm_data *iommu_mm = mm->iommu_mm; + struct iopf_fault *iopf; + struct iopf_group *group; + enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS; - if (!iommu_mm) - return; + group = container_of(work, struct iopf_group, work); + list_for_each_entry(iopf, &group->faults, list) { + /* + * For the moment, errors are sticky: don't handle subsequent + * faults in the group if there is an error. + */ + if (status != IOMMU_PAGE_RESP_SUCCESS) + break; - iommu_free_global_pasid(iommu_mm->pasid); - kfree(iommu_mm); + status = iommu_sva_handle_mm(&iopf->fault, group->domain->mm); + } + + iopf_group_response(group, status); + iopf_free_group(group); +} + +static int iommu_sva_iopf_handler(struct iopf_group *group) +{ + struct iommu_fault_param *fault_param = group->fault_param; + + INIT_WORK(&group->work, iommu_sva_handle_iopf); + if (!queue_work(fault_param->queue->wq, &group->work)) + return -EBUSY; + + return 0; +} + +struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, + struct mm_struct *mm) +{ + const struct iommu_ops *ops = dev_iommu_ops(dev); + struct iommu_domain *domain; + + domain = ops->domain_alloc(IOMMU_DOMAIN_SVA); + if (!domain) + return NULL; + + domain->type = IOMMU_DOMAIN_SVA; + mmgrab(mm); + domain->mm = mm; + domain->owner = ops; + domain->iopf_handler = iommu_sva_iopf_handler; + + return domain; } diff --git a/drivers/iommu/iommu-sva.h b/drivers/iommu/iommu-sva.h deleted file mode 100644 index 54946b5a7caf..000000000000 --- a/drivers/iommu/iommu-sva.h +++ /dev/null @@ -1,71 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * SVA library for IOMMU drivers - */ -#ifndef _IOMMU_SVA_H -#define _IOMMU_SVA_H - -#include - -/* I/O Page fault */ -struct device; -struct iommu_fault; -struct iopf_queue; - -#ifdef CONFIG_IOMMU_SVA -int iommu_queue_iopf(struct iommu_fault *fault, void *cookie); - -int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev); -int iopf_queue_remove_device(struct iopf_queue *queue, - struct device *dev); -int iopf_queue_flush_dev(struct device *dev); -struct iopf_queue *iopf_queue_alloc(const char *name); -void iopf_queue_free(struct iopf_queue *queue); -int iopf_queue_discard_partial(struct iopf_queue *queue); -enum iommu_page_response_code -iommu_sva_handle_iopf(struct iommu_fault *fault, void *data); - -#else /* CONFIG_IOMMU_SVA */ -static inline int iommu_queue_iopf(struct iommu_fault *fault, void *cookie) -{ - return -ENODEV; -} - -static inline int iopf_queue_add_device(struct iopf_queue *queue, - struct device *dev) -{ - return -ENODEV; -} - -static inline int iopf_queue_remove_device(struct iopf_queue *queue, - struct device *dev) -{ - return -ENODEV; -} - -static inline int iopf_queue_flush_dev(struct device *dev) -{ - return -ENODEV; -} - -static inline struct iopf_queue *iopf_queue_alloc(const char *name) -{ - return NULL; -} - -static inline void iopf_queue_free(struct iopf_queue *queue) -{ -} - -static inline int iopf_queue_discard_partial(struct iopf_queue *queue) -{ - return -ENODEV; -} - -static inline enum iommu_page_response_code -iommu_sva_handle_iopf(struct iommu_fault *fault, void *data) -{ - return IOMMU_PAGE_RESP_INVALID; -} -#endif /* CONFIG_IOMMU_SVA */ -#endif /* _IOMMU_SVA_H */ diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d14413916f93..098869007c69 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -36,8 +36,6 @@ #include "dma-iommu.h" #include "iommu-priv.h" -#include "iommu-sva.h" - static struct kset *iommu_group_kset; static DEFINE_IDA(iommu_group_ida); static DEFINE_IDA(iommu_global_pasid_ida); @@ -291,7 +289,7 @@ EXPORT_SYMBOL_GPL(iommu_device_unregister); #if IS_ENABLED(CONFIG_IOMMUFD_TEST) void iommu_device_unregister_bus(struct iommu_device *iommu, - struct bus_type *bus, + const struct bus_type *bus, struct notifier_block *nb) { bus_unregister_notifier(bus, nb); @@ -305,7 +303,8 @@ EXPORT_SYMBOL_GPL(iommu_device_unregister_bus); * some memory to hold a notifier_block. */ int iommu_device_register_bus(struct iommu_device *iommu, - const struct iommu_ops *ops, struct bus_type *bus, + const struct iommu_ops *ops, + const struct bus_type *bus, struct notifier_block *nb) { int err; @@ -463,13 +462,24 @@ static void iommu_deinit_device(struct device *dev) /* * release_device() must stop using any attached domain on the device. - * If there are still other devices in the group they are not effected + * If there are still other devices in the group, they are not affected * by this callback. * - * The IOMMU driver must set the device to either an identity or - * blocking translation and stop using any domain pointer, as it is - * going to be freed. + * If the iommu driver provides release_domain, the core code ensures + * that domain is attached prior to calling release_device. Drivers can + * use this to enforce a translation on the idle iommu. Typically, the + * global static blocked_domain is a good choice. + * + * Otherwise, the iommu driver must set the device to either an identity + * or a blocking translation in release_device() and stop using any + * domain pointer, as it is going to be freed. + * + * Regardless, if a delayed attach never occurred, then the release + * should still avoid touching any hardware configuration either. */ + if (!dev->iommu->attach_deferred && ops->release_domain) + ops->release_domain->ops->attach_dev(ops->release_domain, dev); + if (ops->release_device) ops->release_device(dev); @@ -1248,6 +1258,25 @@ void iommu_group_remove_device(struct device *dev) } EXPORT_SYMBOL_GPL(iommu_group_remove_device); +#if IS_ENABLED(CONFIG_LOCKDEP) && IS_ENABLED(CONFIG_IOMMU_API) +/** + * iommu_group_mutex_assert - Check device group mutex lock + * @dev: the device that has group param set + * + * This function is called by an iommu driver to check whether it holds + * group mutex lock for the given device or not. + * + * Note that this function must be called after device group param is set. + */ +void iommu_group_mutex_assert(struct device *dev) +{ + struct iommu_group *group = dev->iommu_group; + + lockdep_assert_held(&group->mutex); +} +EXPORT_SYMBOL_GPL(iommu_group_mutex_assert); +#endif + static struct device *iommu_group_first_dev(struct iommu_group *group) { lockdep_assert_held(&group->mutex); @@ -1330,217 +1359,6 @@ void iommu_group_put(struct iommu_group *group) } EXPORT_SYMBOL_GPL(iommu_group_put); -/** - * iommu_register_device_fault_handler() - Register a device fault handler - * @dev: the device - * @handler: the fault handler - * @data: private data passed as argument to the handler - * - * When an IOMMU fault event is received, this handler gets called with the - * fault event and data as argument. The handler should return 0 on success. If - * the fault is recoverable (IOMMU_FAULT_PAGE_REQ), the consumer should also - * complete the fault by calling iommu_page_response() with one of the following - * response code: - * - IOMMU_PAGE_RESP_SUCCESS: retry the translation - * - IOMMU_PAGE_RESP_INVALID: terminate the fault - * - IOMMU_PAGE_RESP_FAILURE: terminate the fault and stop reporting - * page faults if possible. - * - * Return 0 if the fault handler was installed successfully, or an error. - */ -int iommu_register_device_fault_handler(struct device *dev, - iommu_dev_fault_handler_t handler, - void *data) -{ - struct dev_iommu *param = dev->iommu; - int ret = 0; - - if (!param) - return -EINVAL; - - mutex_lock(¶m->lock); - /* Only allow one fault handler registered for each device */ - if (param->fault_param) { - ret = -EBUSY; - goto done_unlock; - } - - get_device(dev); - param->fault_param = kzalloc(sizeof(*param->fault_param), GFP_KERNEL); - if (!param->fault_param) { - put_device(dev); - ret = -ENOMEM; - goto done_unlock; - } - param->fault_param->handler = handler; - param->fault_param->data = data; - mutex_init(¶m->fault_param->lock); - INIT_LIST_HEAD(¶m->fault_param->faults); - -done_unlock: - mutex_unlock(¶m->lock); - - return ret; -} -EXPORT_SYMBOL_GPL(iommu_register_device_fault_handler); - -/** - * iommu_unregister_device_fault_handler() - Unregister the device fault handler - * @dev: the device - * - * Remove the device fault handler installed with - * iommu_register_device_fault_handler(). - * - * Return 0 on success, or an error. - */ -int iommu_unregister_device_fault_handler(struct device *dev) -{ - struct dev_iommu *param = dev->iommu; - int ret = 0; - - if (!param) - return -EINVAL; - - mutex_lock(¶m->lock); - - if (!param->fault_param) - goto unlock; - - /* we cannot unregister handler if there are pending faults */ - if (!list_empty(¶m->fault_param->faults)) { - ret = -EBUSY; - goto unlock; - } - - kfree(param->fault_param); - param->fault_param = NULL; - put_device(dev); -unlock: - mutex_unlock(¶m->lock); - - return ret; -} -EXPORT_SYMBOL_GPL(iommu_unregister_device_fault_handler); - -/** - * iommu_report_device_fault() - Report fault event to device driver - * @dev: the device - * @evt: fault event data - * - * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ - * handler. When this function fails and the fault is recoverable, it is the - * caller's responsibility to complete the fault. - * - * Return 0 on success, or an error. - */ -int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) -{ - struct dev_iommu *param = dev->iommu; - struct iommu_fault_event *evt_pending = NULL; - struct iommu_fault_param *fparam; - int ret = 0; - - if (!param || !evt) - return -EINVAL; - - /* we only report device fault if there is a handler registered */ - mutex_lock(¶m->lock); - fparam = param->fault_param; - if (!fparam || !fparam->handler) { - ret = -EINVAL; - goto done_unlock; - } - - if (evt->fault.type == IOMMU_FAULT_PAGE_REQ && - (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) { - evt_pending = kmemdup(evt, sizeof(struct iommu_fault_event), - GFP_KERNEL); - if (!evt_pending) { - ret = -ENOMEM; - goto done_unlock; - } - mutex_lock(&fparam->lock); - list_add_tail(&evt_pending->list, &fparam->faults); - mutex_unlock(&fparam->lock); - } - - ret = fparam->handler(&evt->fault, fparam->data); - if (ret && evt_pending) { - mutex_lock(&fparam->lock); - list_del(&evt_pending->list); - mutex_unlock(&fparam->lock); - kfree(evt_pending); - } -done_unlock: - mutex_unlock(¶m->lock); - return ret; -} -EXPORT_SYMBOL_GPL(iommu_report_device_fault); - -int iommu_page_response(struct device *dev, - struct iommu_page_response *msg) -{ - bool needs_pasid; - int ret = -EINVAL; - struct iommu_fault_event *evt; - struct iommu_fault_page_request *prm; - struct dev_iommu *param = dev->iommu; - const struct iommu_ops *ops = dev_iommu_ops(dev); - bool has_pasid = msg->flags & IOMMU_PAGE_RESP_PASID_VALID; - - if (!ops->page_response) - return -ENODEV; - - if (!param || !param->fault_param) - return -EINVAL; - - if (msg->version != IOMMU_PAGE_RESP_VERSION_1 || - msg->flags & ~IOMMU_PAGE_RESP_PASID_VALID) - return -EINVAL; - - /* Only send response if there is a fault report pending */ - mutex_lock(¶m->fault_param->lock); - if (list_empty(¶m->fault_param->faults)) { - dev_warn_ratelimited(dev, "no pending PRQ, drop response\n"); - goto done_unlock; - } - /* - * Check if we have a matching page request pending to respond, - * otherwise return -EINVAL - */ - list_for_each_entry(evt, ¶m->fault_param->faults, list) { - prm = &evt->fault.prm; - if (prm->grpid != msg->grpid) - continue; - - /* - * If the PASID is required, the corresponding request is - * matched using the group ID, the PASID valid bit and the PASID - * value. Otherwise only the group ID matches request and - * response. - */ - needs_pasid = prm->flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; - if (needs_pasid && (!has_pasid || msg->pasid != prm->pasid)) - continue; - - if (!needs_pasid && has_pasid) { - /* No big deal, just clear it. */ - msg->flags &= ~IOMMU_PAGE_RESP_PASID_VALID; - msg->pasid = 0; - } - - ret = ops->page_response(dev, evt, msg); - list_del(&evt->list); - kfree(evt); - break; - } - -done_unlock: - mutex_unlock(¶m->fault_param->lock); - return ret; -} -EXPORT_SYMBOL_GPL(iommu_page_response); - /** * iommu_group_id - Return ID for a group * @group: the group to ID @@ -2986,7 +2804,7 @@ bool iommu_default_passthrough(void) } EXPORT_SYMBOL_GPL(iommu_default_passthrough); -const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode) +const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode) { const struct iommu_ops *ops = NULL; struct iommu_device *iommu; @@ -3037,7 +2855,7 @@ void iommu_fwspec_free(struct device *dev) } EXPORT_SYMBOL_GPL(iommu_fwspec_free); -int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids) +int iommu_fwspec_add_ids(struct device *dev, const u32 *ids, int num_ids) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); int i, new_num; @@ -3623,26 +3441,6 @@ struct iommu_domain *iommu_get_domain_for_dev_pasid(struct device *dev, } EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev_pasid); -struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, - struct mm_struct *mm) -{ - const struct iommu_ops *ops = dev_iommu_ops(dev); - struct iommu_domain *domain; - - domain = ops->domain_alloc(IOMMU_DOMAIN_SVA); - if (!domain) - return NULL; - - domain->type = IOMMU_DOMAIN_SVA; - mmgrab(mm); - domain->mm = mm; - domain->owner = ops; - domain->iopf_handler = iommu_sva_handle_iopf; - domain->fault_data = mm; - - return domain; -} - ioasid_t iommu_alloc_global_pasid(struct device *dev) { int ret; diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index d30e453d0fb4..d59d0ea2fd21 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -24,24 +24,8 @@ static bool iova_rcache_insert(struct iova_domain *iovad, static unsigned long iova_rcache_get(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn); -static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad); static void free_iova_rcaches(struct iova_domain *iovad); - -unsigned long iova_rcache_range(void) -{ - return PAGE_SIZE << (IOVA_RANGE_CACHE_MAX_SIZE - 1); -} - -static int iova_cpuhp_dead(unsigned int cpu, struct hlist_node *node) -{ - struct iova_domain *iovad; - - iovad = hlist_entry_safe(node, struct iova_domain, cpuhp_dead); - - free_cpu_cached_iovas(cpu, iovad); - return 0; -} - +static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad); static void free_global_cached_iovas(struct iova_domain *iovad); static struct iova *to_iova(struct rb_node *node) @@ -252,54 +236,6 @@ static void free_iova_mem(struct iova *iova) kmem_cache_free(iova_cache, iova); } -int iova_cache_get(void) -{ - mutex_lock(&iova_cache_mutex); - if (!iova_cache_users) { - int ret; - - ret = cpuhp_setup_state_multi(CPUHP_IOMMU_IOVA_DEAD, "iommu/iova:dead", NULL, - iova_cpuhp_dead); - if (ret) { - mutex_unlock(&iova_cache_mutex); - pr_err("Couldn't register cpuhp handler\n"); - return ret; - } - - iova_cache = kmem_cache_create( - "iommu_iova", sizeof(struct iova), 0, - SLAB_HWCACHE_ALIGN, NULL); - if (!iova_cache) { - cpuhp_remove_multi_state(CPUHP_IOMMU_IOVA_DEAD); - mutex_unlock(&iova_cache_mutex); - pr_err("Couldn't create iova cache\n"); - return -ENOMEM; - } - } - - iova_cache_users++; - mutex_unlock(&iova_cache_mutex); - - return 0; -} -EXPORT_SYMBOL_GPL(iova_cache_get); - -void iova_cache_put(void) -{ - mutex_lock(&iova_cache_mutex); - if (WARN_ON(!iova_cache_users)) { - mutex_unlock(&iova_cache_mutex); - return; - } - iova_cache_users--; - if (!iova_cache_users) { - cpuhp_remove_multi_state(CPUHP_IOMMU_IOVA_DEAD); - kmem_cache_destroy(iova_cache); - } - mutex_unlock(&iova_cache_mutex); -} -EXPORT_SYMBOL_GPL(iova_cache_put); - /** * alloc_iova - allocates an iova * @iovad: - iova domain in question @@ -654,11 +590,18 @@ struct iova_rcache { struct delayed_work work; }; +static struct kmem_cache *iova_magazine_cache; + +unsigned long iova_rcache_range(void) +{ + return PAGE_SIZE << (IOVA_RANGE_CACHE_MAX_SIZE - 1); +} + static struct iova_magazine *iova_magazine_alloc(gfp_t flags) { struct iova_magazine *mag; - mag = kmalloc(sizeof(*mag), flags); + mag = kmem_cache_alloc(iova_magazine_cache, flags); if (mag) mag->size = 0; @@ -667,7 +610,7 @@ static struct iova_magazine *iova_magazine_alloc(gfp_t flags) static void iova_magazine_free(struct iova_magazine *mag) { - kfree(mag); + kmem_cache_free(iova_magazine_cache, mag); } static void @@ -990,5 +933,71 @@ static void free_global_cached_iovas(struct iova_domain *iovad) spin_unlock_irqrestore(&rcache->lock, flags); } } + +static int iova_cpuhp_dead(unsigned int cpu, struct hlist_node *node) +{ + struct iova_domain *iovad; + + iovad = hlist_entry_safe(node, struct iova_domain, cpuhp_dead); + + free_cpu_cached_iovas(cpu, iovad); + return 0; +} + +int iova_cache_get(void) +{ + int err = -ENOMEM; + + mutex_lock(&iova_cache_mutex); + if (!iova_cache_users) { + iova_cache = kmem_cache_create("iommu_iova", sizeof(struct iova), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!iova_cache) + goto out_err; + + iova_magazine_cache = kmem_cache_create("iommu_iova_magazine", + sizeof(struct iova_magazine), + 0, SLAB_HWCACHE_ALIGN, NULL); + if (!iova_magazine_cache) + goto out_err; + + err = cpuhp_setup_state_multi(CPUHP_IOMMU_IOVA_DEAD, "iommu/iova:dead", + NULL, iova_cpuhp_dead); + if (err) { + pr_err("IOVA: Couldn't register cpuhp handler: %pe\n", ERR_PTR(err)); + goto out_err; + } + } + + iova_cache_users++; + mutex_unlock(&iova_cache_mutex); + + return 0; + +out_err: + kmem_cache_destroy(iova_cache); + kmem_cache_destroy(iova_magazine_cache); + mutex_unlock(&iova_cache_mutex); + return err; +} +EXPORT_SYMBOL_GPL(iova_cache_get); + +void iova_cache_put(void) +{ + mutex_lock(&iova_cache_mutex); + if (WARN_ON(!iova_cache_users)) { + mutex_unlock(&iova_cache_mutex); + return; + } + iova_cache_users--; + if (!iova_cache_users) { + cpuhp_remove_multi_state(CPUHP_IOMMU_IOVA_DEAD); + kmem_cache_destroy(iova_cache); + kmem_cache_destroy(iova_magazine_cache); + } + mutex_unlock(&iova_cache_mutex); +} +EXPORT_SYMBOL_GPL(iova_cache_put); + MODULE_AUTHOR("Anil S Keshavamurthy "); MODULE_LICENSE("GPL"); diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index ace1fc4bd34b..b657cc09605f 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -709,7 +709,7 @@ static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain, } static int ipmmu_init_platform_device(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct platform_device *ipmmu_pdev; @@ -773,7 +773,7 @@ static bool ipmmu_device_is_allowed(struct device *dev) } static int ipmmu_of_xlate(struct device *dev, - struct of_phandle_args *spec) + const struct of_phandle_args *spec) { if (!ipmmu_device_is_allowed(dev)) return -ENODEV; @@ -1005,7 +1005,6 @@ static const struct of_device_id ipmmu_of_ids[] = { static int ipmmu_probe(struct platform_device *pdev) { struct ipmmu_vmsa_device *mmu; - struct resource *res; int irq; int ret; @@ -1025,8 +1024,7 @@ static int ipmmu_probe(struct platform_device *pdev) return ret; /* Map I/O memory and request IRQ. */ - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - mmu->base = devm_ioremap_resource(&pdev->dev, res); + mmu->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(mmu->base)) return PTR_ERR(mmu->base); @@ -1123,7 +1121,6 @@ static void ipmmu_remove(struct platform_device *pdev) ipmmu_device_reset(mmu); } -#ifdef CONFIG_PM_SLEEP static int ipmmu_resume_noirq(struct device *dev) { struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev); @@ -1153,18 +1150,14 @@ static int ipmmu_resume_noirq(struct device *dev) } static const struct dev_pm_ops ipmmu_pm = { - SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, ipmmu_resume_noirq) + NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, ipmmu_resume_noirq) }; -#define DEV_PM_OPS &ipmmu_pm -#else -#define DEV_PM_OPS NULL -#endif /* CONFIG_PM_SLEEP */ static struct platform_driver ipmmu_driver = { .driver = { .name = "ipmmu-vmsa", - .of_match_table = of_match_ptr(ipmmu_of_ids), - .pm = DEV_PM_OPS, + .of_match_table = ipmmu_of_ids, + .pm = pm_sleep_ptr(&ipmmu_pm), }, .probe = ipmmu_probe, .remove_new = ipmmu_remove, diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c index 83314b9d8f38..ee59647c2050 100644 --- a/drivers/iommu/irq_remapping.c +++ b/drivers/iommu/irq_remapping.c @@ -99,7 +99,8 @@ int __init irq_remapping_prepare(void) if (disable_irq_remap) return -ENOSYS; - if (intel_irq_remap_ops.prepare() == 0) + if (IS_ENABLED(CONFIG_INTEL_IOMMU) && + intel_irq_remap_ops.prepare() == 0) remap_ops = &intel_irq_remap_ops; else if (IS_ENABLED(CONFIG_AMD_IOMMU) && amd_iommu_irq_ops.prepare() == 0) diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index f86af9815d6f..989e0869d805 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -598,7 +598,7 @@ static void print_ctx_regs(void __iomem *base, int ctx) static int insert_iommu_master(struct device *dev, struct msm_iommu_dev **iommu, - struct of_phandle_args *spec) + const struct of_phandle_args *spec) { struct msm_iommu_ctx_dev *master = dev_iommu_priv_get(dev); int sid; @@ -626,7 +626,7 @@ static int insert_iommu_master(struct device *dev, } static int qcom_iommu_of_xlate(struct device *dev, - struct of_phandle_args *spec) + const struct of_phandle_args *spec) { struct msm_iommu_dev *iommu = NULL, *iter; unsigned long flags; diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 7abe9e85a570..b8c47f18bc26 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -957,7 +957,8 @@ static struct iommu_group *mtk_iommu_device_group(struct device *dev) return group; } -static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int mtk_iommu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { struct platform_device *m4updev; @@ -1264,7 +1265,7 @@ static int mtk_iommu_probe(struct platform_device *pdev) data->plat_data = of_device_get_match_data(dev); /* Protect memory. HW will access here while translation fault.*/ - protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL); + protect = devm_kcalloc(dev, 2, MTK_PROTECT_PA_ALIGN, GFP_KERNEL); if (!protect) return -ENOMEM; data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 25b41222abae..a9fa2a54dc9b 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -398,7 +398,8 @@ static const struct iommu_ops mtk_iommu_v1_ops; * MTK generation one iommu HW only support one iommu domain, and all the client * sharing the same iova address space. */ -static int mtk_iommu_v1_create_mapping(struct device *dev, struct of_phandle_args *args) +static int mtk_iommu_v1_create_mapping(struct device *dev, + const struct of_phandle_args *args) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct mtk_iommu_v1_data *data; @@ -621,8 +622,8 @@ static int mtk_iommu_v1_probe(struct platform_device *pdev) data->dev = dev; /* Protect memory. HW will access here while translation fault.*/ - protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, - GFP_KERNEL | GFP_DMA); + protect = devm_kcalloc(dev, 2, MTK_PROTECT_PA_ALIGN, + GFP_KERNEL | GFP_DMA); if (!protect) return -ENOMEM; data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 719652b60840..3afe0b48a48d 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -29,7 +29,7 @@ static int of_iommu_xlate(struct device *dev, !of_device_is_available(iommu_spec->np)) return -ENODEV; - ret = iommu_fwspec_init(dev, &iommu_spec->np->fwnode, ops); + ret = iommu_fwspec_init(dev, fwnode, ops); if (ret) return ret; /* diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 2685861c0a12..da79d9f4cf63 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -1140,7 +1140,7 @@ static void rk_iommu_release_device(struct device *dev) } static int rk_iommu_of_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct platform_device *iommu_dev; struct rk_iommudata *data; diff --git a/drivers/iommu/sprd-iommu.c b/drivers/iommu/sprd-iommu.c index 537359f10997..ba53571a8239 100644 --- a/drivers/iommu/sprd-iommu.c +++ b/drivers/iommu/sprd-iommu.c @@ -390,7 +390,8 @@ static struct iommu_device *sprd_iommu_probe_device(struct device *dev) return &sdev->iommu; } -static int sprd_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int sprd_iommu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { struct platform_device *pdev; diff --git a/drivers/iommu/sun50i-iommu.c b/drivers/iommu/sun50i-iommu.c index 41484a5a399b..decd52cba998 100644 --- a/drivers/iommu/sun50i-iommu.c +++ b/drivers/iommu/sun50i-iommu.c @@ -819,7 +819,7 @@ static struct iommu_device *sun50i_iommu_probe_device(struct device *dev) } static int sun50i_iommu_of_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct platform_device *iommu_pdev = of_find_device_by_node(args->np); unsigned id = args->args[0]; diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 310871728ab4..14e525bd0d9b 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -830,7 +830,7 @@ static struct tegra_smmu *tegra_smmu_find(struct device_node *np) } static int tegra_smmu_configure(struct tegra_smmu *smmu, struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { const struct iommu_ops *ops = smmu->iommu.ops; int err; @@ -959,7 +959,7 @@ static struct iommu_group *tegra_smmu_device_group(struct device *dev) } static int tegra_smmu_of_xlate(struct device *dev, - struct of_phandle_args *args) + const struct of_phandle_args *args) { struct platform_device *iommu_pdev = of_find_device_by_node(args->np); struct tegra_mc *mc = platform_get_drvdata(iommu_pdev); diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index 34db37fd9675..04048f64a2c0 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -1051,7 +1051,8 @@ static struct iommu_group *viommu_device_group(struct device *dev) return generic_device_group(dev); } -static int viommu_of_xlate(struct device *dev, struct of_phandle_args *args) +static int viommu_of_xlate(struct device *dev, + const struct of_phandle_args *args) { return iommu_fwspec_add_ids(dev, args->args, 1); } diff --git a/drivers/leds/rgb/leds-qcom-lpg.c b/drivers/leds/rgb/leds-qcom-lpg.c index 156b73d1f4a2..0a7acf59a420 100644 --- a/drivers/leds/rgb/leds-qcom-lpg.c +++ b/drivers/leds/rgb/leds-qcom-lpg.c @@ -77,7 +77,7 @@ struct lpg { struct mutex lock; - struct pwm_chip pwm; + struct pwm_chip *pwm; const struct lpg_data *data; @@ -978,7 +978,7 @@ static int lpg_pattern_mc_clear(struct led_classdev *cdev) static inline struct lpg *lpg_pwm_from_chip(struct pwm_chip *chip) { - return container_of(chip, struct lpg, pwm); + return pwmchip_get_drvdata(chip); } static int lpg_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) @@ -1093,13 +1093,17 @@ static const struct pwm_ops lpg_pwm_ops = { static int lpg_add_pwm(struct lpg *lpg) { + struct pwm_chip *chip; int ret; - lpg->pwm.dev = lpg->dev; - lpg->pwm.npwm = lpg->num_channels; - lpg->pwm.ops = &lpg_pwm_ops; + lpg->pwm = chip = devm_pwmchip_alloc(lpg->dev, lpg->num_channels, 0); + if (IS_ERR(chip)) + return PTR_ERR(chip); - ret = devm_pwmchip_add(lpg->dev, &lpg->pwm); + chip->ops = &lpg_pwm_ops; + pwmchip_set_drvdata(chip, lpg); + + ret = devm_pwmchip_add(lpg->dev, chip); if (ret) dev_err_probe(lpg->dev, ret, "failed to add PWM chip\n"); diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig index 86ac46319342..657b227527cc 100644 --- a/drivers/md/Kconfig +++ b/drivers/md/Kconfig @@ -669,4 +669,6 @@ config DM_USER If unsure, say N. +source "drivers/md/dm-vdo/Kconfig" + endif # MD diff --git a/drivers/md/Makefile b/drivers/md/Makefile index d11dc1169681..996aaf0b5d27 100644 --- a/drivers/md/Makefile +++ b/drivers/md/Makefile @@ -69,6 +69,7 @@ obj-$(CONFIG_DM_ZERO) += dm-zero.o obj-$(CONFIG_DM_RAID) += dm-raid.o obj-$(CONFIG_DM_THIN_PROVISIONING) += dm-thin-pool.o obj-$(CONFIG_DM_VERITY) += dm-verity.o +obj-$(CONFIG_DM_VDO) += dm-vdo/ obj-$(CONFIG_DM_CACHE) += dm-cache.o obj-$(CONFIG_DM_CACHE_SMQ) += dm-cache-smq.o obj-$(CONFIG_DM_EBS) += dm-ebs.o diff --git a/drivers/md/dm-bio-prison-v1.c b/drivers/md/dm-bio-prison-v1.c index 9ab32abe5ed4..bca0f39e15b8 100644 --- a/drivers/md/dm-bio-prison-v1.c +++ b/drivers/md/dm-bio-prison-v1.c @@ -489,5 +489,5 @@ module_init(dm_bio_prison_init); module_exit(dm_bio_prison_exit); MODULE_DESCRIPTION(DM_NAME " bio prison"); -MODULE_AUTHOR("Joe Thornber "); +MODULE_AUTHOR("Joe Thornber "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index 13c65b7e1ed6..098bf526136c 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -1292,7 +1292,8 @@ static void dmio_complete(unsigned long error, void *context) } static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector, - unsigned int n_sectors, unsigned int offset) + unsigned int n_sectors, unsigned int offset, + unsigned short ioprio) { int r; struct dm_io_request io_req = { @@ -1315,7 +1316,7 @@ static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector, io_req.mem.ptr.vma = (char *)b->data + offset; } - r = dm_io(&io_req, 1, ®ion, NULL); + r = dm_io(&io_req, 1, ®ion, NULL, ioprio); if (unlikely(r)) b->end_io(b, errno_to_blk_status(r)); } @@ -1331,7 +1332,8 @@ static void bio_complete(struct bio *bio) } static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector, - unsigned int n_sectors, unsigned int offset) + unsigned int n_sectors, unsigned int offset, + unsigned short ioprio) { struct bio *bio; char *ptr; @@ -1339,13 +1341,14 @@ static void use_bio(struct dm_buffer *b, enum req_op op, sector_t sector, bio = bio_kmalloc(1, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOWARN); if (!bio) { - use_dmio(b, op, sector, n_sectors, offset); + use_dmio(b, op, sector, n_sectors, offset, ioprio); return; } bio_init(bio, b->c->bdev, bio->bi_inline_vecs, 1, op); bio->bi_iter.bi_sector = sector; bio->bi_end_io = bio_complete; bio->bi_private = b; + bio->bi_ioprio = ioprio; ptr = (char *)b->data + offset; len = n_sectors << SECTOR_SHIFT; @@ -1368,7 +1371,7 @@ static inline sector_t block_to_sector(struct dm_bufio_client *c, sector_t block return sector; } -static void submit_io(struct dm_buffer *b, enum req_op op, +static void submit_io(struct dm_buffer *b, enum req_op op, unsigned short ioprio, void (*end_io)(struct dm_buffer *, blk_status_t)) { unsigned int n_sectors; @@ -1398,9 +1401,9 @@ static void submit_io(struct dm_buffer *b, enum req_op op, } if (b->data_mode != DATA_MODE_VMALLOC) - use_bio(b, op, sector, n_sectors, offset); + use_bio(b, op, sector, n_sectors, offset, ioprio); else - use_dmio(b, op, sector, n_sectors, offset); + use_dmio(b, op, sector, n_sectors, offset, ioprio); } /* @@ -1456,7 +1459,7 @@ static void __write_dirty_buffer(struct dm_buffer *b, b->write_end = b->dirty_end; if (!write_list) - submit_io(b, REQ_OP_WRITE, write_endio); + submit_io(b, REQ_OP_WRITE, IOPRIO_DEFAULT, write_endio); else list_add_tail(&b->write_list, write_list); } @@ -1470,7 +1473,7 @@ static void __flush_write_list(struct list_head *write_list) struct dm_buffer *b = list_entry(write_list->next, struct dm_buffer, write_list); list_del(&b->write_list); - submit_io(b, REQ_OP_WRITE, write_endio); + submit_io(b, REQ_OP_WRITE, IOPRIO_DEFAULT, write_endio); cond_resched(); } blk_finish_plug(&plug); @@ -1852,7 +1855,8 @@ static void read_endio(struct dm_buffer *b, blk_status_t status) * and uses dm_bufio_mark_buffer_dirty to write new data back). */ static void *new_read(struct dm_bufio_client *c, sector_t block, - enum new_flag nf, struct dm_buffer **bp) + enum new_flag nf, struct dm_buffer **bp, + unsigned short ioprio) { int need_submit = 0; struct dm_buffer *b; @@ -1905,7 +1909,7 @@ static void *new_read(struct dm_bufio_client *c, sector_t block, return NULL; if (need_submit) - submit_io(b, REQ_OP_READ, read_endio); + submit_io(b, REQ_OP_READ, ioprio, read_endio); if (nf != NF_GET) /* we already tested this condition above */ wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); @@ -1926,32 +1930,46 @@ static void *new_read(struct dm_bufio_client *c, sector_t block, void *dm_bufio_get(struct dm_bufio_client *c, sector_t block, struct dm_buffer **bp) { - return new_read(c, block, NF_GET, bp); + return new_read(c, block, NF_GET, bp, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_get); -void *dm_bufio_read(struct dm_bufio_client *c, sector_t block, - struct dm_buffer **bp) +static void *__dm_bufio_read(struct dm_bufio_client *c, sector_t block, + struct dm_buffer **bp, unsigned short ioprio) { if (WARN_ON_ONCE(dm_bufio_in_request())) return ERR_PTR(-EINVAL); - return new_read(c, block, NF_READ, bp); + return new_read(c, block, NF_READ, bp, ioprio); +} + +void *dm_bufio_read(struct dm_bufio_client *c, sector_t block, + struct dm_buffer **bp) +{ + return __dm_bufio_read(c, block, bp, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_read); +void *dm_bufio_read_with_ioprio(struct dm_bufio_client *c, sector_t block, + struct dm_buffer **bp, unsigned short ioprio) +{ + return __dm_bufio_read(c, block, bp, ioprio); +} +EXPORT_SYMBOL_GPL(dm_bufio_read_with_ioprio); + void *dm_bufio_new(struct dm_bufio_client *c, sector_t block, struct dm_buffer **bp) { if (WARN_ON_ONCE(dm_bufio_in_request())) return ERR_PTR(-EINVAL); - return new_read(c, block, NF_FRESH, bp); + return new_read(c, block, NF_FRESH, bp, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_new); -void dm_bufio_prefetch(struct dm_bufio_client *c, - sector_t block, unsigned int n_blocks) +static void __dm_bufio_prefetch(struct dm_bufio_client *c, + sector_t block, unsigned int n_blocks, + unsigned short ioprio) { struct blk_plug plug; @@ -1987,7 +2005,7 @@ void dm_bufio_prefetch(struct dm_bufio_client *c, dm_bufio_unlock(c); if (need_submit) - submit_io(b, REQ_OP_READ, read_endio); + submit_io(b, REQ_OP_READ, ioprio, read_endio); dm_bufio_release(b); cond_resched(); @@ -2002,8 +2020,20 @@ void dm_bufio_prefetch(struct dm_bufio_client *c, flush_plug: blk_finish_plug(&plug); } + +void dm_bufio_prefetch(struct dm_bufio_client *c, sector_t block, unsigned int n_blocks) +{ + return __dm_bufio_prefetch(c, block, n_blocks, IOPRIO_DEFAULT); +} EXPORT_SYMBOL_GPL(dm_bufio_prefetch); +void dm_bufio_prefetch_with_ioprio(struct dm_bufio_client *c, sector_t block, + unsigned int n_blocks, unsigned short ioprio) +{ + return __dm_bufio_prefetch(c, block, n_blocks, ioprio); +} +EXPORT_SYMBOL_GPL(dm_bufio_prefetch_with_ioprio); + void dm_bufio_release(struct dm_buffer *b) { struct dm_bufio_client *c = b->c; @@ -2167,7 +2197,7 @@ int dm_bufio_issue_flush(struct dm_bufio_client *c) if (WARN_ON_ONCE(dm_bufio_in_request())) return -EINVAL; - return dm_io(&io_req, 1, &io_reg, NULL); + return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_issue_flush); @@ -2191,7 +2221,7 @@ int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t c if (WARN_ON_ONCE(dm_bufio_in_request())) return -EINVAL; /* discards are optional */ - return dm_io(&io_req, 1, &io_reg, NULL); + return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_issue_discard); @@ -2968,6 +2998,6 @@ MODULE_PARM_DESC(allocated_vmalloc_bytes, "Memory allocated with vmalloc"); module_param_named(current_allocated_bytes, dm_bufio_current_allocated, ulong, 0444); MODULE_PARM_DESC(current_allocated_bytes, "Memory currently used by the cache"); -MODULE_AUTHOR("Mikulas Patocka "); +MODULE_AUTHOR("Mikulas Patocka "); MODULE_DESCRIPTION(DM_NAME " buffered I/O library"); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-cache-policy-smq.c b/drivers/md/dm-cache-policy-smq.c index 8bd2ad743d9a..2ed894155cab 100644 --- a/drivers/md/dm-cache-policy-smq.c +++ b/drivers/md/dm-cache-policy-smq.c @@ -1947,7 +1947,7 @@ static void __exit smq_exit(void) module_init(smq_init); module_exit(smq_exit); -MODULE_AUTHOR("Joe Thornber "); +MODULE_AUTHOR("Joe Thornber "); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("smq cache policy"); diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 59445763e55a..9a74c6316c5d 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -1688,6 +1688,7 @@ retry: GFP_NOIO, &cc->bs); clone->bi_private = io; clone->bi_end_io = crypt_endio; + clone->bi_ioprio = io->base_bio->bi_ioprio; remaining_size = size; @@ -1964,7 +1965,6 @@ continue_locked: schedule(); - set_current_state(TASK_RUNNING); spin_lock_irq(&cc->write_thread_lock); goto continue_locked; @@ -2296,7 +2296,11 @@ static void kcryptd_queue_crypt(struct dm_crypt_io *io) * irqs_disabled(): the kernel may run some IO completion from the idle thread, but * it is being executed with irqs disabled. */ - if (!(in_hardirq() || irqs_disabled())) { + if (in_hardirq() || irqs_disabled()) { + INIT_WORK(&io->work, kcryptd_crypt); + queue_work(system_bh_wq, &io->work); + return; + } else { kcryptd_crypt(&io->work); return; } diff --git a/drivers/md/dm-dust.c b/drivers/md/dm-dust.c index 12a377e06d02..1a33820c9f46 100644 --- a/drivers/md/dm-dust.c +++ b/drivers/md/dm-dust.c @@ -573,5 +573,5 @@ static struct target_type dust_target = { module_dm(dust); MODULE_DESCRIPTION(DM_NAME " dust test target"); -MODULE_AUTHOR("Bryan Gurney "); +MODULE_AUTHOR("Bryan Gurney "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-ebs-target.c b/drivers/md/dm-ebs-target.c index 435b45201f4d..b70d4016c2ac 100644 --- a/drivers/md/dm-ebs-target.c +++ b/drivers/md/dm-ebs-target.c @@ -454,6 +454,6 @@ static struct target_type ebs_target = { }; module_dm(ebs); -MODULE_AUTHOR("Heinz Mauelshagen "); +MODULE_AUTHOR("Heinz Mauelshagen "); MODULE_DESCRIPTION(DM_NAME " emulated block size target"); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-flakey.c b/drivers/md/dm-flakey.c index 7916ed9f10e8..731467d4ed10 100644 --- a/drivers/md/dm-flakey.c +++ b/drivers/md/dm-flakey.c @@ -690,5 +690,5 @@ static struct target_type flakey_target = { module_dm(flakey); MODULE_DESCRIPTION(DM_NAME " flakey target"); -MODULE_AUTHOR("Joe Thornber "); +MODULE_AUTHOR("Joe Thornber "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 1fc901df84eb..d822ab2f739b 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -555,7 +555,7 @@ static int sync_rw_sb(struct dm_integrity_c *ic, blk_opf_t opf) } } - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) return r; @@ -1073,7 +1073,7 @@ static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf, io_loc.sector = ic->start + SB_SECTORS + sector; io_loc.count = n_sectors; - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { dm_integrity_io_error(ic, (opf & REQ_OP_MASK) == REQ_OP_READ ? "reading journal" : "writing journal", r); @@ -1190,7 +1190,7 @@ static void copy_from_journal(struct dm_integrity_c *ic, unsigned int section, u io_loc.sector = target; io_loc.count = n_sectors; - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { WARN_ONCE(1, "asynchronous dm_io failed: %d", r); fn(-1UL, data); @@ -1519,7 +1519,7 @@ static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_dat fr.io_reg.count = 0, fr.ic = ic; init_completion(&fr.comp); - r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL); + r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL, IOPRIO_DEFAULT); BUG_ON(r); } @@ -1727,7 +1727,7 @@ static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checks io_loc.sector = sector; io_loc.count = ic->sectors_per_block; - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { dio->bi_status = errno_to_blk_status(r); goto free_ret; @@ -2806,7 +2806,7 @@ next_chunk: io_loc.sector = get_data_sector(ic, area, offset); io_loc.count = n_sectors; - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { dm_integrity_io_error(ic, "reading data", r); goto err; @@ -3485,6 +3485,7 @@ static void dm_integrity_io_hints(struct dm_target *ti, struct queue_limits *lim blk_limits_io_min(limits, ic->sectors_per_block << SECTOR_SHIFT); limits->dma_alignment = limits->logical_block_size - 1; } + limits->max_integrity_segments = USHRT_MAX; } static void calculate_journal_section_size(struct dm_integrity_c *ic) @@ -3652,7 +3653,6 @@ static void dm_integrity_set(struct dm_target *ti, struct dm_integrity_c *ic) bi.interval_exp = ic->sb->log2_sectors_per_block + SECTOR_SHIFT; blk_integrity_register(disk, &bi); - blk_queue_max_integrity_segments(disk->queue, UINT_MAX); } static void dm_integrity_free_page_list(struct page_list *pl) diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c index f053ce245814..7409490259d1 100644 --- a/drivers/md/dm-io.c +++ b/drivers/md/dm-io.c @@ -305,7 +305,7 @@ static void km_dp_init(struct dpages *dp, void *data) */ static void do_region(const blk_opf_t opf, unsigned int region, struct dm_io_region *where, struct dpages *dp, - struct io *io) + struct io *io, unsigned short ioprio) { struct bio *bio; struct page *page; @@ -354,6 +354,7 @@ static void do_region(const blk_opf_t opf, unsigned int region, &io->client->bios); bio->bi_iter.bi_sector = where->sector + (where->count - remaining); bio->bi_end_io = endio; + bio->bi_ioprio = ioprio; store_io_and_region_in_bio(bio, io, region); if (op == REQ_OP_DISCARD || op == REQ_OP_WRITE_ZEROES) { @@ -383,7 +384,7 @@ static void do_region(const blk_opf_t opf, unsigned int region, static void dispatch_io(blk_opf_t opf, unsigned int num_regions, struct dm_io_region *where, struct dpages *dp, - struct io *io, int sync) + struct io *io, int sync, unsigned short ioprio) { int i; struct dpages old_pages = *dp; @@ -400,7 +401,7 @@ static void dispatch_io(blk_opf_t opf, unsigned int num_regions, for (i = 0; i < num_regions; i++) { *dp = old_pages; if (where[i].count || (opf & REQ_PREFLUSH)) - do_region(opf, i, where + i, dp, io); + do_region(opf, i, where + i, dp, io, ioprio); } /* @@ -425,7 +426,7 @@ static void sync_io_complete(unsigned long error, void *context) static int sync_io(struct dm_io_client *client, unsigned int num_regions, struct dm_io_region *where, blk_opf_t opf, struct dpages *dp, - unsigned long *error_bits) + unsigned long *error_bits, unsigned short ioprio) { struct io *io; struct sync_io sio; @@ -447,7 +448,7 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions, io->vma_invalidate_address = dp->vma_invalidate_address; io->vma_invalidate_size = dp->vma_invalidate_size; - dispatch_io(opf, num_regions, where, dp, io, 1); + dispatch_io(opf, num_regions, where, dp, io, 1, ioprio); wait_for_completion_io(&sio.wait); @@ -459,7 +460,8 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions, static int async_io(struct dm_io_client *client, unsigned int num_regions, struct dm_io_region *where, blk_opf_t opf, - struct dpages *dp, io_notify_fn fn, void *context) + struct dpages *dp, io_notify_fn fn, void *context, + unsigned short ioprio) { struct io *io; @@ -479,7 +481,7 @@ static int async_io(struct dm_io_client *client, unsigned int num_regions, io->vma_invalidate_address = dp->vma_invalidate_address; io->vma_invalidate_size = dp->vma_invalidate_size; - dispatch_io(opf, num_regions, where, dp, io, 0); + dispatch_io(opf, num_regions, where, dp, io, 0, ioprio); return 0; } @@ -521,7 +523,8 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp, } int dm_io(struct dm_io_request *io_req, unsigned int num_regions, - struct dm_io_region *where, unsigned long *sync_error_bits) + struct dm_io_region *where, unsigned long *sync_error_bits, + unsigned short ioprio) { int r; struct dpages dp; @@ -532,11 +535,11 @@ int dm_io(struct dm_io_request *io_req, unsigned int num_regions, if (!io_req->notify.fn) return sync_io(io_req->client, num_regions, where, - io_req->bi_opf, &dp, sync_error_bits); + io_req->bi_opf, &dp, sync_error_bits, ioprio); return async_io(io_req->client, num_regions, where, io_req->bi_opf, &dp, io_req->notify.fn, - io_req->notify.context); + io_req->notify.context, ioprio); } EXPORT_SYMBOL(dm_io); diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c index 3b1ad7127cb8..c2c07bfa6471 100644 --- a/drivers/md/dm-ioctl.c +++ b/drivers/md/dm-ioctl.c @@ -25,7 +25,7 @@ #include #define DM_MSG_PREFIX "ioctl" -#define DM_DRIVER_EMAIL "dm-devel@redhat.com" +#define DM_DRIVER_EMAIL "dm-devel@lists.linux.dev" struct dm_file { /* diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c index 36bcfdccae04..6ea75436a433 100644 --- a/drivers/md/dm-kcopyd.c +++ b/drivers/md/dm-kcopyd.c @@ -578,9 +578,9 @@ static int run_io_job(struct kcopyd_job *job) io_job_start(job->kc->throttle); if (job->op == REQ_OP_READ) - r = dm_io(&io_req, 1, &job->source, NULL); + r = dm_io(&io_req, 1, &job->source, NULL, IOPRIO_DEFAULT); else - r = dm_io(&io_req, job->num_dests, job->dests, NULL); + r = dm_io(&io_req, job->num_dests, job->dests, NULL, IOPRIO_DEFAULT); return r; } diff --git a/drivers/md/dm-log-userspace-base.c b/drivers/md/dm-log-userspace-base.c index 7e4f27e86150..9fbb4b48fb2b 100644 --- a/drivers/md/dm-log-userspace-base.c +++ b/drivers/md/dm-log-userspace-base.c @@ -926,5 +926,5 @@ module_init(userspace_dirty_log_init); module_exit(userspace_dirty_log_exit); MODULE_DESCRIPTION(DM_NAME " userspace dirty log link"); -MODULE_AUTHOR("Jonathan Brassow "); +MODULE_AUTHOR("Jonathan Brassow "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c index f9f84236dfcd..9d85d045f9d9 100644 --- a/drivers/md/dm-log.c +++ b/drivers/md/dm-log.c @@ -300,7 +300,7 @@ static int rw_header(struct log_c *lc, enum req_op op) { lc->io_req.bi_opf = op; - return dm_io(&lc->io_req, 1, &lc->header_location, NULL); + return dm_io(&lc->io_req, 1, &lc->header_location, NULL, IOPRIO_DEFAULT); } static int flush_header(struct log_c *lc) @@ -313,7 +313,7 @@ static int flush_header(struct log_c *lc) lc->io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; - return dm_io(&lc->io_req, 1, &null_location, NULL); + return dm_io(&lc->io_req, 1, &null_location, NULL, IOPRIO_DEFAULT); } static int read_header(struct log_c *log) @@ -908,5 +908,5 @@ module_init(dm_dirty_log_init); module_exit(dm_dirty_log_exit); MODULE_DESCRIPTION(DM_NAME " dirty region log"); -MODULE_AUTHOR("Joe Thornber, Heinz Mauelshagen "); +MODULE_AUTHOR("Joe Thornber, Heinz Mauelshagen "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c index bea3cda9938e..05d1328d1811 100644 --- a/drivers/md/dm-mpath.c +++ b/drivers/md/dm-mpath.c @@ -2266,5 +2266,5 @@ module_param_named(queue_if_no_path_timeout_secs, queue_if_no_path_timeout_secs, MODULE_PARM_DESC(queue_if_no_path_timeout_secs, "No available paths queue IO timeout in seconds"); MODULE_DESCRIPTION(DM_NAME " multipath target"); -MODULE_AUTHOR("Sistina Software "); +MODULE_AUTHOR("Sistina Software "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-ps-round-robin.c b/drivers/md/dm-ps-round-robin.c index 0f04b673597a..d1745b123dc1 100644 --- a/drivers/md/dm-ps-round-robin.c +++ b/drivers/md/dm-ps-round-robin.c @@ -240,5 +240,5 @@ module_init(dm_rr_init); module_exit(dm_rr_exit); MODULE_DESCRIPTION(DM_NAME " round-robin multipath path selector"); -MODULE_AUTHOR("Sistina Software "); +MODULE_AUTHOR("Sistina Software "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c index 17e9af60bbf7..abe88d1e6735 100644 --- a/drivers/md/dm-raid.c +++ b/drivers/md/dm-raid.c @@ -3331,14 +3331,14 @@ static int raid_map(struct dm_target *ti, struct bio *bio) struct mddev *mddev = &rs->md; /* - * If we're reshaping to add disk(s)), ti->len and + * If we're reshaping to add disk(s), ti->len and * mddev->array_sectors will differ during the process * (ti->len > mddev->array_sectors), so we have to requeue * bios with addresses > mddev->array_sectors here or * there will occur accesses past EOD of the component * data images thus erroring the raid set. */ - if (unlikely(bio_end_sector(bio) > mddev->array_sectors)) + if (unlikely(bio_has_data(bio) && bio_end_sector(bio) > mddev->array_sectors)) return DM_MAPIO_REQUEUE; if (unlikely(!md_handle_request(mddev, bio))) @@ -4142,6 +4142,6 @@ MODULE_ALIAS("dm-raid10"); MODULE_ALIAS("dm-raid4"); MODULE_ALIAS("dm-raid5"); MODULE_ALIAS("dm-raid6"); -MODULE_AUTHOR("Neil Brown "); -MODULE_AUTHOR("Heinz Mauelshagen "); +MODULE_AUTHOR("Neil Brown "); +MODULE_AUTHOR("Heinz Mauelshagen "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c index ddcb2bc4a617..9511dae5b556 100644 --- a/drivers/md/dm-raid1.c +++ b/drivers/md/dm-raid1.c @@ -278,7 +278,7 @@ static int mirror_flush(struct dm_target *ti) } error_bits = -1; - dm_io(&io_req, ms->nr_mirrors, io, &error_bits); + dm_io(&io_req, ms->nr_mirrors, io, &error_bits, IOPRIO_DEFAULT); if (unlikely(error_bits != 0)) { for (i = 0; i < ms->nr_mirrors; i++) if (test_bit(i, &error_bits)) @@ -554,7 +554,7 @@ static void read_async_bio(struct mirror *m, struct bio *bio) map_region(&io, m, bio); bio_set_m(bio, m); - BUG_ON(dm_io(&io_req, 1, &io, NULL)); + BUG_ON(dm_io(&io_req, 1, &io, NULL, IOPRIO_DEFAULT)); } static inline int region_in_sync(struct mirror_set *ms, region_t region, @@ -681,7 +681,7 @@ static void do_write(struct mirror_set *ms, struct bio *bio) */ bio_set_m(bio, get_default_mirror(ms)); - BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL)); + BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL, IOPRIO_DEFAULT)); } static void do_writes(struct mirror_set *ms, struct bio_list *writes) diff --git a/drivers/md/dm-region-hash.c b/drivers/md/dm-region-hash.c index 852cfa37d48a..a4550975c27d 100644 --- a/drivers/md/dm-region-hash.c +++ b/drivers/md/dm-region-hash.c @@ -723,5 +723,5 @@ void dm_rh_start_recovery(struct dm_region_hash *rh) EXPORT_SYMBOL_GPL(dm_rh_start_recovery); MODULE_DESCRIPTION(DM_NAME " region hash"); -MODULE_AUTHOR("Joe Thornber/Heinz Mauelshagen "); +MODULE_AUTHOR("Joe Thornber/Heinz Mauelshagen "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-snap-persistent.c b/drivers/md/dm-snap-persistent.c index 15649921f2a9..568d10842b1f 100644 --- a/drivers/md/dm-snap-persistent.c +++ b/drivers/md/dm-snap-persistent.c @@ -223,7 +223,7 @@ static void do_metadata(struct work_struct *work) { struct mdata_req *req = container_of(work, struct mdata_req, work); - req->result = dm_io(req->io_req, 1, req->where, NULL); + req->result = dm_io(req->io_req, 1, req->where, NULL, IOPRIO_DEFAULT); } /* @@ -247,7 +247,7 @@ static int chunk_io(struct pstore *ps, void *area, chunk_t chunk, blk_opf_t opf, struct mdata_req req; if (!metadata) - return dm_io(&io_req, 1, &where, NULL); + return dm_io(&io_req, 1, &where, NULL, IOPRIO_DEFAULT); req.where = &where; req.io_req = &io_req; diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c index 07c7f9795b10..4793ad2aa1f7 100644 --- a/drivers/md/dm-thin.c +++ b/drivers/md/dm-thin.c @@ -453,12 +453,13 @@ static int bio_detain(struct pool *pool, struct dm_cell_key *key, struct bio *bi cell_prealloc = dm_bio_prison_alloc_cell(pool->prison, GFP_NOIO); r = dm_bio_detain(pool->prison, key, bio, cell_prealloc, cell_result); - if (r) + if (r) { /* * We reused an old cell; we can get rid of * the new one. */ dm_bio_prison_free_cell(pool->prison, cell_prealloc); + } return r; } @@ -707,9 +708,10 @@ static void get_bio_block_range(struct thin_c *tc, struct bio *bio, (void) sector_div(e, pool->sectors_per_block); } - if (e < b) + if (e < b) { /* Can happen if the bio is within a single block. */ e = b; + } *begin = b; *end = e; @@ -721,13 +723,14 @@ static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block) sector_t bi_sector = bio->bi_iter.bi_sector; bio_set_dev(bio, tc->pool_dev->bdev); - if (block_size_is_power_of_two(pool)) + if (block_size_is_power_of_two(pool)) { bio->bi_iter.bi_sector = (block << pool->sectors_per_block_shift) | (bi_sector & (pool->sectors_per_block - 1)); - else + } else { bio->bi_iter.bi_sector = (block * pool->sectors_per_block) + sector_div(bi_sector, pool->sectors_per_block); + } } static void remap_to_origin(struct thin_c *tc, struct bio *bio) @@ -1401,9 +1404,10 @@ static void schedule_zero(struct thin_c *tc, dm_block_t virt_block, if (pool->pf.zero_new_blocks) { if (io_overwrites_block(pool, bio)) remap_and_issue_overwrite(tc, bio, data_block, m); - else + else { ll_zero(tc, m, data_block * pool->sectors_per_block, (data_block + 1) * pool->sectors_per_block); + } } else process_prepared_mapping(m); } @@ -1416,17 +1420,17 @@ static void schedule_external_copy(struct thin_c *tc, dm_block_t virt_block, sector_t virt_block_begin = virt_block * pool->sectors_per_block; sector_t virt_block_end = (virt_block + 1) * pool->sectors_per_block; - if (virt_block_end <= tc->origin_size) + if (virt_block_end <= tc->origin_size) { schedule_copy(tc, virt_block, tc->origin_dev, virt_block, data_dest, cell, bio, pool->sectors_per_block); - else if (virt_block_begin < tc->origin_size) + } else if (virt_block_begin < tc->origin_size) { schedule_copy(tc, virt_block, tc->origin_dev, virt_block, data_dest, cell, bio, tc->origin_size - virt_block_begin); - else + } else schedule_zero(tc, virt_block, data_dest, cell, bio); } @@ -4560,5 +4564,5 @@ module_param_named(no_space_timeout, no_space_timeout_secs, uint, 0644); MODULE_PARM_DESC(no_space_timeout, "Out of data space queue IO timeout in seconds"); MODULE_DESCRIPTION(DM_NAME " thin provisioning target"); -MODULE_AUTHOR("Joe Thornber "); +MODULE_AUTHOR("Joe Thornber "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-vdo/Kconfig b/drivers/md/dm-vdo/Kconfig new file mode 100644 index 000000000000..111ecd2c2a24 --- /dev/null +++ b/drivers/md/dm-vdo/Kconfig @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config DM_VDO + tristate "VDO: deduplication and compression target" + depends on 64BIT + depends on BLK_DEV_DM + select DM_BUFIO + select LZ4_COMPRESS + select LZ4_DECOMPRESS + help + This device mapper target presents a block device with + deduplication, compression and thin-provisioning. + + To compile this code as a module, choose M here: the module will + be called dm-vdo. + + If unsure, say N. diff --git a/drivers/md/dm-vdo/Makefile b/drivers/md/dm-vdo/Makefile new file mode 100644 index 000000000000..33e09abc6acd --- /dev/null +++ b/drivers/md/dm-vdo/Makefile @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: GPL-2.0-only + +ccflags-y := -I$(srctree)/$(src) -I$(srctree)/$(src)/indexer + +obj-$(CONFIG_DM_VDO) += dm-vdo.o + +dm-vdo-objs := \ + action-manager.o \ + admin-state.o \ + block-map.o \ + completion.o \ + data-vio.o \ + dedupe.o \ + dm-vdo-target.o \ + dump.o \ + encodings.o \ + errors.o \ + flush.o \ + funnel-queue.o \ + funnel-workqueue.o \ + int-map.o \ + io-submitter.o \ + logger.o \ + logical-zone.o \ + memory-alloc.o \ + message-stats.o \ + murmurhash3.o \ + packer.o \ + permassert.o \ + physical-zone.o \ + priority-table.o \ + recovery-journal.o \ + repair.o \ + slab-depot.o \ + status-codes.o \ + string-utils.o \ + thread-device.o \ + thread-registry.o \ + thread-utils.o \ + vdo.o \ + vio.o \ + wait-queue.o \ + indexer/chapter-index.o \ + indexer/config.o \ + indexer/delta-index.o \ + indexer/funnel-requestqueue.o \ + indexer/geometry.o \ + indexer/index.o \ + indexer/index-layout.o \ + indexer/index-page-map.o \ + indexer/index-session.o \ + indexer/io-factory.o \ + indexer/open-chapter.o \ + indexer/radix-sort.o \ + indexer/sparse-cache.o \ + indexer/volume.o \ + indexer/volume-index.o diff --git a/drivers/md/dm-vdo/action-manager.c b/drivers/md/dm-vdo/action-manager.c new file mode 100644 index 000000000000..a0e5e7077d13 --- /dev/null +++ b/drivers/md/dm-vdo/action-manager.c @@ -0,0 +1,388 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "action-manager.h" + +#include "memory-alloc.h" +#include "permassert.h" + +#include "admin-state.h" +#include "completion.h" +#include "status-codes.h" +#include "types.h" +#include "vdo.h" + +/** + * struct action - An action to be performed in each of a set of zones. + * @in_use: Whether this structure is in use. + * @operation: The admin operation associated with this action. + * @preamble: The method to run on the initiator thread before the action is applied to each zone. + * @zone_action: The action to be performed in each zone. + * @conclusion: The method to run on the initiator thread after the action is applied to each zone. + * @parent: The object to notify when the action is complete. + * @context: The action specific context. + * @next: The action to perform after this one. + */ +struct action { + bool in_use; + const struct admin_state_code *operation; + vdo_action_preamble_fn preamble; + vdo_zone_action_fn zone_action; + vdo_action_conclusion_fn conclusion; + struct vdo_completion *parent; + void *context; + struct action *next; +}; + +/** + * struct action_manager - Definition of an action manager. + * @completion: The completion for performing actions. + * @state: The state of this action manager. + * @actions: The two action slots. + * @current_action: The current action slot. + * @zones: The number of zones in which an action is to be applied. + * @Scheduler: A function to schedule a default next action. + * @get_zone_thread_id: A function to get the id of the thread on which to apply an action to a + * zone. + * @initiator_thread_id: The ID of the thread on which actions may be initiated. + * @context: Opaque data associated with this action manager. + * @acting_zone: The zone currently being acted upon. + */ +struct action_manager { + struct vdo_completion completion; + struct admin_state state; + struct action actions[2]; + struct action *current_action; + zone_count_t zones; + vdo_action_scheduler_fn scheduler; + vdo_zone_thread_getter_fn get_zone_thread_id; + thread_id_t initiator_thread_id; + void *context; + zone_count_t acting_zone; +}; + +static inline struct action_manager *as_action_manager(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_ACTION_COMPLETION); + return container_of(completion, struct action_manager, completion); +} + +/* Implements vdo_action_scheduler_fn. */ +static bool no_default_action(void *context __always_unused) +{ + return false; +} + +/* Implements vdo_action_preamble_fn. */ +static void no_preamble(void *context __always_unused, struct vdo_completion *completion) +{ + vdo_finish_completion(completion); +} + +/* Implements vdo_action_conclusion_fn. */ +static int no_conclusion(void *context __always_unused) +{ + return VDO_SUCCESS; +} + +/** + * vdo_make_action_manager() - Make an action manager. + * @zones: The number of zones to which actions will be applied. + * @get_zone_thread_id: A function to get the thread id associated with a zone. + * @initiator_thread_id: The thread on which actions may initiated. + * @context: The object which holds the per-zone context for the action. + * @scheduler: A function to schedule a next action after an action concludes if there is no + * pending action (may be NULL). + * @vdo: The vdo used to initialize completions. + * @manager_ptr: A pointer to hold the new action manager. + * + * Return: VDO_SUCCESS or an error code. + */ +int vdo_make_action_manager(zone_count_t zones, + vdo_zone_thread_getter_fn get_zone_thread_id, + thread_id_t initiator_thread_id, void *context, + vdo_action_scheduler_fn scheduler, struct vdo *vdo, + struct action_manager **manager_ptr) +{ + struct action_manager *manager; + int result = vdo_allocate(1, struct action_manager, __func__, &manager); + + if (result != VDO_SUCCESS) + return result; + + *manager = (struct action_manager) { + .zones = zones, + .scheduler = + ((scheduler == NULL) ? no_default_action : scheduler), + .get_zone_thread_id = get_zone_thread_id, + .initiator_thread_id = initiator_thread_id, + .context = context, + }; + + manager->actions[0].next = &manager->actions[1]; + manager->current_action = manager->actions[1].next = + &manager->actions[0]; + vdo_set_admin_state_code(&manager->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + vdo_initialize_completion(&manager->completion, vdo, VDO_ACTION_COMPLETION); + *manager_ptr = manager; + return VDO_SUCCESS; +} + +const struct admin_state_code *vdo_get_current_manager_operation(struct action_manager *manager) +{ + return vdo_get_admin_state_code(&manager->state); +} + +void *vdo_get_current_action_context(struct action_manager *manager) +{ + return manager->current_action->in_use ? manager->current_action->context : NULL; +} + +static void finish_action_callback(struct vdo_completion *completion); +static void apply_to_zone(struct vdo_completion *completion); + +static thread_id_t get_acting_zone_thread_id(struct action_manager *manager) +{ + return manager->get_zone_thread_id(manager->context, manager->acting_zone); +} + +static void preserve_error(struct vdo_completion *completion) +{ + if (completion->parent != NULL) + vdo_set_completion_result(completion->parent, completion->result); + + vdo_reset_completion(completion); + vdo_run_completion(completion); +} + +static void prepare_for_next_zone(struct action_manager *manager) +{ + vdo_prepare_completion_for_requeue(&manager->completion, apply_to_zone, + preserve_error, + get_acting_zone_thread_id(manager), + manager->current_action->parent); +} + +static void prepare_for_conclusion(struct action_manager *manager) +{ + vdo_prepare_completion_for_requeue(&manager->completion, finish_action_callback, + preserve_error, manager->initiator_thread_id, + manager->current_action->parent); +} + +static void apply_to_zone(struct vdo_completion *completion) +{ + zone_count_t zone; + struct action_manager *manager = as_action_manager(completion); + + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == get_acting_zone_thread_id(manager)), + "%s() called on acting zones's thread", __func__); + + zone = manager->acting_zone++; + if (manager->acting_zone == manager->zones) { + /* + * We are about to apply to the last zone. Once that is finished, we're done, so go + * back to the initiator thread and finish up. + */ + prepare_for_conclusion(manager); + } else { + /* Prepare to come back on the next zone */ + prepare_for_next_zone(manager); + } + + manager->current_action->zone_action(manager->context, zone, completion); +} + +static void handle_preamble_error(struct vdo_completion *completion) +{ + /* Skip the zone actions since the preamble failed. */ + completion->callback = finish_action_callback; + preserve_error(completion); +} + +static void launch_current_action(struct action_manager *manager) +{ + struct action *action = manager->current_action; + int result = vdo_start_operation(&manager->state, action->operation); + + if (result != VDO_SUCCESS) { + if (action->parent != NULL) + vdo_set_completion_result(action->parent, result); + + /* We aren't going to run the preamble, so don't run the conclusion */ + action->conclusion = no_conclusion; + finish_action_callback(&manager->completion); + return; + } + + if (action->zone_action == NULL) { + prepare_for_conclusion(manager); + } else { + manager->acting_zone = 0; + vdo_prepare_completion_for_requeue(&manager->completion, apply_to_zone, + handle_preamble_error, + get_acting_zone_thread_id(manager), + manager->current_action->parent); + } + + action->preamble(manager->context, &manager->completion); +} + +/** + * vdo_schedule_default_action() - Attempt to schedule the default action. + * @manager: The action manager. + * + * If the manager is not operating normally, the action will not be scheduled. + * + * Return: true if an action was scheduled. + */ +bool vdo_schedule_default_action(struct action_manager *manager) +{ + /* Don't schedule a default action if we are operating or not in normal operation. */ + const struct admin_state_code *code = vdo_get_current_manager_operation(manager); + + return ((code == VDO_ADMIN_STATE_NORMAL_OPERATION) && + manager->scheduler(manager->context)); +} + +static void finish_action_callback(struct vdo_completion *completion) +{ + bool has_next_action; + int result; + struct action_manager *manager = as_action_manager(completion); + struct action action = *(manager->current_action); + + manager->current_action->in_use = false; + manager->current_action = manager->current_action->next; + + /* + * We need to check this now to avoid use-after-free issues if running the conclusion or + * notifying the parent results in the manager being freed. + */ + has_next_action = + (manager->current_action->in_use || vdo_schedule_default_action(manager)); + result = action.conclusion(manager->context); + vdo_finish_operation(&manager->state, VDO_SUCCESS); + if (action.parent != NULL) + vdo_continue_completion(action.parent, result); + + if (has_next_action) + launch_current_action(manager); +} + +/** + * vdo_schedule_action() - Schedule an action to be applied to all zones. + * @manager: The action manager to schedule the action on. + * @preamble: A method to be invoked on the initiator thread once this action is started but before + * applying to each zone; may be NULL. + * @action: The action to apply to each zone; may be NULL. + * @conclusion: A method to be invoked back on the initiator thread once the action has been + * applied to all zones; may be NULL. + * @parent: The object to notify once the action is complete or if the action can not be scheduled; + * may be NULL. + * + * The action will be launched immediately if there is no current action, or as soon as the current + * action completes. If there is already a pending action, this action will not be scheduled, and, + * if it has a parent, that parent will be notified. At least one of the preamble, action, or + * conclusion must not be NULL. + * + * Return: true if the action was scheduled. + */ +bool vdo_schedule_action(struct action_manager *manager, vdo_action_preamble_fn preamble, + vdo_zone_action_fn action, vdo_action_conclusion_fn conclusion, + struct vdo_completion *parent) +{ + return vdo_schedule_operation(manager, VDO_ADMIN_STATE_OPERATING, preamble, + action, conclusion, parent); +} + +/** + * vdo_schedule_operation() - Schedule an operation to be applied to all zones. + * @manager: The action manager to schedule the action on. + * @operation: The operation this action will perform + * @preamble: A method to be invoked on the initiator thread once this action is started but before + * applying to each zone; may be NULL. + * @action: The action to apply to each zone; may be NULL. + * @conclusion: A method to be invoked back on the initiator thread once the action has been + * applied to all zones; may be NULL. + * @parent: The object to notify once the action is complete or if the action can not be scheduled; + * may be NULL. + * + * The operation's action will be launched immediately if there is no current action, or as soon as + * the current action completes. If there is already a pending action, this operation will not be + * scheduled, and, if it has a parent, that parent will be notified. At least one of the preamble, + * action, or conclusion must not be NULL. + * + * Return: true if the action was scheduled. + */ +bool vdo_schedule_operation(struct action_manager *manager, + const struct admin_state_code *operation, + vdo_action_preamble_fn preamble, vdo_zone_action_fn action, + vdo_action_conclusion_fn conclusion, + struct vdo_completion *parent) +{ + return vdo_schedule_operation_with_context(manager, operation, preamble, action, + conclusion, NULL, parent); +} + +/** + * vdo_schedule_operation_with_context() - Schedule an operation on all zones. + * @manager: The action manager to schedule the action on. + * @operation: The operation this action will perform. + * @preamble: A method to be invoked on the initiator thread once this action is started but before + * applying to each zone; may be NULL. + * @action: The action to apply to each zone; may be NULL. + * @conclusion: A method to be invoked back on the initiator thread once the action has been + * applied to all zones; may be NULL. + * @context: An action-specific context which may be retrieved via + * vdo_get_current_action_context(); may be NULL. + * @parent: The object to notify once the action is complete or if the action can not be scheduled; + * may be NULL. + * + * The operation's action will be launched immediately if there is no current action, or as soon as + * the current action completes. If there is already a pending action, this operation will not be + * scheduled, and, if it has a parent, that parent will be notified. At least one of the preamble, + * action, or conclusion must not be NULL. + * + * Return: true if the action was scheduled + */ +bool vdo_schedule_operation_with_context(struct action_manager *manager, + const struct admin_state_code *operation, + vdo_action_preamble_fn preamble, + vdo_zone_action_fn action, + vdo_action_conclusion_fn conclusion, + void *context, struct vdo_completion *parent) +{ + struct action *current_action; + + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == manager->initiator_thread_id), + "action initiated from correct thread"); + if (!manager->current_action->in_use) { + current_action = manager->current_action; + } else if (!manager->current_action->next->in_use) { + current_action = manager->current_action->next; + } else { + if (parent != NULL) + vdo_continue_completion(parent, VDO_COMPONENT_BUSY); + + return false; + } + + *current_action = (struct action) { + .in_use = true, + .operation = operation, + .preamble = (preamble == NULL) ? no_preamble : preamble, + .zone_action = action, + .conclusion = (conclusion == NULL) ? no_conclusion : conclusion, + .context = context, + .parent = parent, + .next = current_action->next, + }; + + if (current_action == manager->current_action) + launch_current_action(manager); + + return true; +} diff --git a/drivers/md/dm-vdo/action-manager.h b/drivers/md/dm-vdo/action-manager.h new file mode 100644 index 000000000000..b0a8d3ddf3db --- /dev/null +++ b/drivers/md/dm-vdo/action-manager.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_ACTION_MANAGER_H +#define VDO_ACTION_MANAGER_H + +#include "admin-state.h" +#include "types.h" + +/* + * An action_manager provides a generic mechanism for applying actions to multi-zone entities (such + * as the block map or slab depot). Each action manager is tied to a specific context for which it + * manages actions. The manager ensures that only one action is active on that context at a time, + * and supports at most one pending action. Calls to schedule an action when there is already a + * pending action will result in VDO_COMPONENT_BUSY errors. Actions may only be submitted to the + * action manager from a single thread (which thread is determined when the action manager is + * constructed). + * + * A scheduled action consists of four components: + * + * preamble + * an optional method to be run on the initiator thread before applying the action to all zones + * zone_action + * an optional method to be applied to each of the zones + * conclusion + * an optional method to be run on the initiator thread once the per-zone method has been + * applied to all zones + * parent + * an optional completion to be finished once the conclusion is done + * + * At least one of the three methods must be provided. + */ + +/* + * A function which is to be applied asynchronously to a set of zones. + * @context: The object which holds the per-zone context for the action + * @zone_number: The number of zone to which the action is being applied + * @parent: The object to notify when the action is complete + */ +typedef void (*vdo_zone_action_fn)(void *context, zone_count_t zone_number, + struct vdo_completion *parent); + +/* + * A function which is to be applied asynchronously on an action manager's initiator thread as the + * preamble of an action. + * @context: The object which holds the per-zone context for the action + * @parent: The object to notify when the action is complete + */ +typedef void (*vdo_action_preamble_fn)(void *context, struct vdo_completion *parent); + +/* + * A function which will run on the action manager's initiator thread as the conclusion of an + * action. + * @context: The object which holds the per-zone context for the action + * + * Return: VDO_SUCCESS or an error + */ +typedef int (*vdo_action_conclusion_fn)(void *context); + +/* + * A function to schedule an action. + * @context: The object which holds the per-zone context for the action + * + * Return: true if an action was scheduled + */ +typedef bool (*vdo_action_scheduler_fn)(void *context); + +/* + * A function to get the id of the thread associated with a given zone. + * @context: The action context + * @zone_number: The number of the zone for which the thread ID is desired + */ +typedef thread_id_t (*vdo_zone_thread_getter_fn)(void *context, zone_count_t zone_number); + +struct action_manager; + +int __must_check vdo_make_action_manager(zone_count_t zones, + vdo_zone_thread_getter_fn get_zone_thread_id, + thread_id_t initiator_thread_id, void *context, + vdo_action_scheduler_fn scheduler, + struct vdo *vdo, + struct action_manager **manager_ptr); + +const struct admin_state_code *__must_check +vdo_get_current_manager_operation(struct action_manager *manager); + +void * __must_check vdo_get_current_action_context(struct action_manager *manager); + +bool vdo_schedule_default_action(struct action_manager *manager); + +bool vdo_schedule_action(struct action_manager *manager, vdo_action_preamble_fn preamble, + vdo_zone_action_fn action, vdo_action_conclusion_fn conclusion, + struct vdo_completion *parent); + +bool vdo_schedule_operation(struct action_manager *manager, + const struct admin_state_code *operation, + vdo_action_preamble_fn preamble, vdo_zone_action_fn action, + vdo_action_conclusion_fn conclusion, + struct vdo_completion *parent); + +bool vdo_schedule_operation_with_context(struct action_manager *manager, + const struct admin_state_code *operation, + vdo_action_preamble_fn preamble, + vdo_zone_action_fn action, + vdo_action_conclusion_fn conclusion, + void *context, struct vdo_completion *parent); + +#endif /* VDO_ACTION_MANAGER_H */ diff --git a/drivers/md/dm-vdo/admin-state.c b/drivers/md/dm-vdo/admin-state.c new file mode 100644 index 000000000000..3f9dba525154 --- /dev/null +++ b/drivers/md/dm-vdo/admin-state.c @@ -0,0 +1,506 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "admin-state.h" + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "completion.h" +#include "types.h" + +static const struct admin_state_code VDO_CODE_NORMAL_OPERATION = { + .name = "VDO_ADMIN_STATE_NORMAL_OPERATION", + .normal = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_NORMAL_OPERATION = &VDO_CODE_NORMAL_OPERATION; +static const struct admin_state_code VDO_CODE_OPERATING = { + .name = "VDO_ADMIN_STATE_OPERATING", + .normal = true, + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_OPERATING = &VDO_CODE_OPERATING; +static const struct admin_state_code VDO_CODE_FORMATTING = { + .name = "VDO_ADMIN_STATE_FORMATTING", + .operating = true, + .loading = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_FORMATTING = &VDO_CODE_FORMATTING; +static const struct admin_state_code VDO_CODE_PRE_LOADING = { + .name = "VDO_ADMIN_STATE_PRE_LOADING", + .operating = true, + .loading = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_PRE_LOADING = &VDO_CODE_PRE_LOADING; +static const struct admin_state_code VDO_CODE_PRE_LOADED = { + .name = "VDO_ADMIN_STATE_PRE_LOADED", +}; +const struct admin_state_code *VDO_ADMIN_STATE_PRE_LOADED = &VDO_CODE_PRE_LOADED; +static const struct admin_state_code VDO_CODE_LOADING = { + .name = "VDO_ADMIN_STATE_LOADING", + .normal = true, + .operating = true, + .loading = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_LOADING = &VDO_CODE_LOADING; +static const struct admin_state_code VDO_CODE_LOADING_FOR_RECOVERY = { + .name = "VDO_ADMIN_STATE_LOADING_FOR_RECOVERY", + .operating = true, + .loading = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_LOADING_FOR_RECOVERY = + &VDO_CODE_LOADING_FOR_RECOVERY; +static const struct admin_state_code VDO_CODE_LOADING_FOR_REBUILD = { + .name = "VDO_ADMIN_STATE_LOADING_FOR_REBUILD", + .operating = true, + .loading = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_LOADING_FOR_REBUILD = &VDO_CODE_LOADING_FOR_REBUILD; +static const struct admin_state_code VDO_CODE_WAITING_FOR_RECOVERY = { + .name = "VDO_ADMIN_STATE_WAITING_FOR_RECOVERY", + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_WAITING_FOR_RECOVERY = + &VDO_CODE_WAITING_FOR_RECOVERY; +static const struct admin_state_code VDO_CODE_NEW = { + .name = "VDO_ADMIN_STATE_NEW", + .quiescent = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_NEW = &VDO_CODE_NEW; +static const struct admin_state_code VDO_CODE_INITIALIZED = { + .name = "VDO_ADMIN_STATE_INITIALIZED", +}; +const struct admin_state_code *VDO_ADMIN_STATE_INITIALIZED = &VDO_CODE_INITIALIZED; +static const struct admin_state_code VDO_CODE_RECOVERING = { + .name = "VDO_ADMIN_STATE_RECOVERING", + .draining = true, + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_RECOVERING = &VDO_CODE_RECOVERING; +static const struct admin_state_code VDO_CODE_REBUILDING = { + .name = "VDO_ADMIN_STATE_REBUILDING", + .draining = true, + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_REBUILDING = &VDO_CODE_REBUILDING; +static const struct admin_state_code VDO_CODE_SAVING = { + .name = "VDO_ADMIN_STATE_SAVING", + .draining = true, + .quiescing = true, + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_SAVING = &VDO_CODE_SAVING; +static const struct admin_state_code VDO_CODE_SAVED = { + .name = "VDO_ADMIN_STATE_SAVED", + .quiescent = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_SAVED = &VDO_CODE_SAVED; +static const struct admin_state_code VDO_CODE_SCRUBBING = { + .name = "VDO_ADMIN_STATE_SCRUBBING", + .draining = true, + .loading = true, + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_SCRUBBING = &VDO_CODE_SCRUBBING; +static const struct admin_state_code VDO_CODE_SAVE_FOR_SCRUBBING = { + .name = "VDO_ADMIN_STATE_SAVE_FOR_SCRUBBING", + .draining = true, + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_SAVE_FOR_SCRUBBING = &VDO_CODE_SAVE_FOR_SCRUBBING; +static const struct admin_state_code VDO_CODE_STOPPING = { + .name = "VDO_ADMIN_STATE_STOPPING", + .draining = true, + .quiescing = true, + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_STOPPING = &VDO_CODE_STOPPING; +static const struct admin_state_code VDO_CODE_STOPPED = { + .name = "VDO_ADMIN_STATE_STOPPED", + .quiescent = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_STOPPED = &VDO_CODE_STOPPED; +static const struct admin_state_code VDO_CODE_SUSPENDING = { + .name = "VDO_ADMIN_STATE_SUSPENDING", + .draining = true, + .quiescing = true, + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_SUSPENDING = &VDO_CODE_SUSPENDING; +static const struct admin_state_code VDO_CODE_SUSPENDED = { + .name = "VDO_ADMIN_STATE_SUSPENDED", + .quiescent = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_SUSPENDED = &VDO_CODE_SUSPENDED; +static const struct admin_state_code VDO_CODE_SUSPENDED_OPERATION = { + .name = "VDO_ADMIN_STATE_SUSPENDED_OPERATION", + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_SUSPENDED_OPERATION = &VDO_CODE_SUSPENDED_OPERATION; +static const struct admin_state_code VDO_CODE_RESUMING = { + .name = "VDO_ADMIN_STATE_RESUMING", + .operating = true, +}; +const struct admin_state_code *VDO_ADMIN_STATE_RESUMING = &VDO_CODE_RESUMING; + +/** + * get_next_state() - Determine the state which should be set after a given operation completes + * based on the operation and the current state. + * @operation The operation to be started. + * + * Return: The state to set when the operation completes or NULL if the operation can not be + * started in the current state. + */ +static const struct admin_state_code *get_next_state(const struct admin_state *state, + const struct admin_state_code *operation) +{ + const struct admin_state_code *code = vdo_get_admin_state_code(state); + + if (code->operating) + return NULL; + + if (operation == VDO_ADMIN_STATE_SAVING) + return (code == VDO_ADMIN_STATE_NORMAL_OPERATION ? VDO_ADMIN_STATE_SAVED : NULL); + + if (operation == VDO_ADMIN_STATE_SUSPENDING) { + return (code == VDO_ADMIN_STATE_NORMAL_OPERATION + ? VDO_ADMIN_STATE_SUSPENDED + : NULL); + } + + if (operation == VDO_ADMIN_STATE_STOPPING) + return (code == VDO_ADMIN_STATE_NORMAL_OPERATION ? VDO_ADMIN_STATE_STOPPED : NULL); + + if (operation == VDO_ADMIN_STATE_PRE_LOADING) + return (code == VDO_ADMIN_STATE_INITIALIZED ? VDO_ADMIN_STATE_PRE_LOADED : NULL); + + if (operation == VDO_ADMIN_STATE_SUSPENDED_OPERATION) { + return (((code == VDO_ADMIN_STATE_SUSPENDED) || + (code == VDO_ADMIN_STATE_SAVED)) ? code : NULL); + } + + return VDO_ADMIN_STATE_NORMAL_OPERATION; +} + +/** + * vdo_finish_operation() - Finish the current operation. + * + * Will notify the operation waiter if there is one. This method should be used for operations + * started with vdo_start_operation(). For operations which were started with vdo_start_draining(), + * use vdo_finish_draining() instead. + * + * Return: true if there was an operation to finish. + */ +bool vdo_finish_operation(struct admin_state *state, int result) +{ + if (!vdo_get_admin_state_code(state)->operating) + return false; + + state->complete = state->starting; + if (state->waiter != NULL) + vdo_set_completion_result(state->waiter, result); + + if (!state->starting) { + vdo_set_admin_state_code(state, state->next_state); + if (state->waiter != NULL) + vdo_launch_completion(vdo_forget(state->waiter)); + } + + return true; +} + +/** + * begin_operation() - Begin an operation if it may be started given the current state. + * @waiter A completion to notify when the operation is complete; may be NULL. + * @initiator The vdo_admin_initiator_fn to call if the operation may begin; may be NULL. + * + * Return: VDO_SUCCESS or an error. + */ +static int __must_check begin_operation(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, + vdo_admin_initiator_fn initiator) +{ + int result; + const struct admin_state_code *next_state = get_next_state(state, operation); + + if (next_state == NULL) { + result = vdo_log_error_strerror(VDO_INVALID_ADMIN_STATE, + "Can't start %s from %s", + operation->name, + vdo_get_admin_state_code(state)->name); + } else if (state->waiter != NULL) { + result = vdo_log_error_strerror(VDO_COMPONENT_BUSY, + "Can't start %s with extant waiter", + operation->name); + } else { + state->waiter = waiter; + state->next_state = next_state; + vdo_set_admin_state_code(state, operation); + if (initiator != NULL) { + state->starting = true; + initiator(state); + state->starting = false; + if (state->complete) + vdo_finish_operation(state, VDO_SUCCESS); + } + + return VDO_SUCCESS; + } + + if (waiter != NULL) + vdo_continue_completion(waiter, result); + + return result; +} + +/** + * start_operation() - Start an operation if it may be started given the current state. + * @waiter A completion to notify when the operation is complete. + * @initiator The vdo_admin_initiator_fn to call if the operation may begin; may be NULL. + * + * Return: true if the operation was started. + */ +static inline bool __must_check start_operation(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, + vdo_admin_initiator_fn initiator) +{ + return (begin_operation(state, operation, waiter, initiator) == VDO_SUCCESS); +} + +/** + * check_code() - Check the result of a state validation. + * @valid true if the code is of an appropriate type. + * @code The code which failed to be of the correct type. + * @what What the code failed to be, for logging. + * @waiter The completion to notify of the error; may be NULL. + * + * If the result failed, log an invalid state error and, if there is a waiter, notify it. + * + * Return: The result of the check. + */ +static bool check_code(bool valid, const struct admin_state_code *code, const char *what, + struct vdo_completion *waiter) +{ + int result; + + if (valid) + return true; + + result = vdo_log_error_strerror(VDO_INVALID_ADMIN_STATE, + "%s is not a %s", code->name, what); + if (waiter != NULL) + vdo_continue_completion(waiter, result); + + return false; +} + +/** + * assert_vdo_drain_operation() - Check that an operation is a drain. + * @waiter The completion to finish with an error if the operation is not a drain. + * + * Return: true if the specified operation is a drain. + */ +static bool __must_check assert_vdo_drain_operation(const struct admin_state_code *operation, + struct vdo_completion *waiter) +{ + return check_code(operation->draining, operation, "drain operation", waiter); +} + +/** + * vdo_start_draining() - Initiate a drain operation if the current state permits it. + * @operation The type of drain to initiate. + * @waiter The completion to notify when the drain is complete. + * @initiator The vdo_admin_initiator_fn to call if the operation may begin; may be NULL. + * + * Return: true if the drain was initiated, if not the waiter will be notified. + */ +bool vdo_start_draining(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, vdo_admin_initiator_fn initiator) +{ + const struct admin_state_code *code = vdo_get_admin_state_code(state); + + if (!assert_vdo_drain_operation(operation, waiter)) + return false; + + if (code->quiescent) { + vdo_launch_completion(waiter); + return false; + } + + if (!code->normal) { + vdo_log_error_strerror(VDO_INVALID_ADMIN_STATE, "can't start %s from %s", + operation->name, code->name); + vdo_continue_completion(waiter, VDO_INVALID_ADMIN_STATE); + return false; + } + + return start_operation(state, operation, waiter, initiator); +} + +/** + * vdo_finish_draining() - Finish a drain operation if one was in progress. + * + * Return: true if the state was draining; will notify the waiter if so. + */ +bool vdo_finish_draining(struct admin_state *state) +{ + return vdo_finish_draining_with_result(state, VDO_SUCCESS); +} + +/** + * vdo_finish_draining_with_result() - Finish a drain operation with a status code. + * + * Return: true if the state was draining; will notify the waiter if so. + */ +bool vdo_finish_draining_with_result(struct admin_state *state, int result) +{ + return (vdo_is_state_draining(state) && vdo_finish_operation(state, result)); +} + +/** + * vdo_assert_load_operation() - Check that an operation is a load. + * @waiter The completion to finish with an error if the operation is not a load. + * + * Return: true if the specified operation is a load. + */ +bool vdo_assert_load_operation(const struct admin_state_code *operation, + struct vdo_completion *waiter) +{ + return check_code(operation->loading, operation, "load operation", waiter); +} + +/** + * vdo_start_loading() - Initiate a load operation if the current state permits it. + * @operation The type of load to initiate. + * @waiter The completion to notify when the load is complete (may be NULL). + * @initiator The vdo_admin_initiator_fn to call if the operation may begin; may be NULL. + * + * Return: true if the load was initiated, if not the waiter will be notified. + */ +bool vdo_start_loading(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, vdo_admin_initiator_fn initiator) +{ + return (vdo_assert_load_operation(operation, waiter) && + start_operation(state, operation, waiter, initiator)); +} + +/** + * vdo_finish_loading() - Finish a load operation if one was in progress. + * + * Return: true if the state was loading; will notify the waiter if so. + */ +bool vdo_finish_loading(struct admin_state *state) +{ + return vdo_finish_loading_with_result(state, VDO_SUCCESS); +} + +/** + * vdo_finish_loading_with_result() - Finish a load operation with a status code. + * @result The result of the load operation. + * + * Return: true if the state was loading; will notify the waiter if so. + */ +bool vdo_finish_loading_with_result(struct admin_state *state, int result) +{ + return (vdo_is_state_loading(state) && vdo_finish_operation(state, result)); +} + +/** + * assert_vdo_resume_operation() - Check whether an admin_state_code is a resume operation. + * @waiter The completion to notify if the operation is not a resume operation; may be NULL. + * + * Return: true if the code is a resume operation. + */ +static bool __must_check assert_vdo_resume_operation(const struct admin_state_code *operation, + struct vdo_completion *waiter) +{ + return check_code(operation == VDO_ADMIN_STATE_RESUMING, operation, + "resume operation", waiter); +} + +/** + * vdo_start_resuming() - Initiate a resume operation if the current state permits it. + * @operation The type of resume to start. + * @waiter The completion to notify when the resume is complete (may be NULL). + * @initiator The vdo_admin_initiator_fn to call if the operation may begin; may be NULL. + * + * Return: true if the resume was initiated, if not the waiter will be notified. + */ +bool vdo_start_resuming(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, vdo_admin_initiator_fn initiator) +{ + return (assert_vdo_resume_operation(operation, waiter) && + start_operation(state, operation, waiter, initiator)); +} + +/** + * vdo_finish_resuming() - Finish a resume operation if one was in progress. + * + * Return: true if the state was resuming; will notify the waiter if so. + */ +bool vdo_finish_resuming(struct admin_state *state) +{ + return vdo_finish_resuming_with_result(state, VDO_SUCCESS); +} + +/** + * vdo_finish_resuming_with_result() - Finish a resume operation with a status code. + * @result The result of the resume operation. + * + * Return: true if the state was resuming; will notify the waiter if so. + */ +bool vdo_finish_resuming_with_result(struct admin_state *state, int result) +{ + return (vdo_is_state_resuming(state) && vdo_finish_operation(state, result)); +} + +/** + * vdo_resume_if_quiescent() - Change the state to normal operation if the current state is + * quiescent. + * + * Return: VDO_SUCCESS if the state resumed, VDO_INVALID_ADMIN_STATE otherwise. + */ +int vdo_resume_if_quiescent(struct admin_state *state) +{ + if (!vdo_is_state_quiescent(state)) + return VDO_INVALID_ADMIN_STATE; + + vdo_set_admin_state_code(state, VDO_ADMIN_STATE_NORMAL_OPERATION); + return VDO_SUCCESS; +} + +/** + * vdo_start_operation() - Attempt to start an operation. + * + * Return: VDO_SUCCESS if the operation was started, VDO_INVALID_ADMIN_STATE if not + */ +int vdo_start_operation(struct admin_state *state, + const struct admin_state_code *operation) +{ + return vdo_start_operation_with_waiter(state, operation, NULL, NULL); +} + +/** + * vdo_start_operation_with_waiter() - Attempt to start an operation. + * @waiter the completion to notify when the operation completes or fails to start; may be NULL. + * @initiator The vdo_admin_initiator_fn to call if the operation may begin; may be NULL. + * + * Return: VDO_SUCCESS if the operation was started, VDO_INVALID_ADMIN_STATE if not + */ +int vdo_start_operation_with_waiter(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, + vdo_admin_initiator_fn initiator) +{ + return (check_code(operation->operating, operation, "operation", waiter) ? + begin_operation(state, operation, waiter, initiator) : + VDO_INVALID_ADMIN_STATE); +} diff --git a/drivers/md/dm-vdo/admin-state.h b/drivers/md/dm-vdo/admin-state.h new file mode 100644 index 000000000000..a7d6ac2c30a6 --- /dev/null +++ b/drivers/md/dm-vdo/admin-state.h @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_ADMIN_STATE_H +#define VDO_ADMIN_STATE_H + +#include "completion.h" +#include "types.h" + +struct admin_state_code { + const char *name; + /* Normal operation, data_vios may be active */ + bool normal; + /* I/O is draining, new requests should not start */ + bool draining; + /* This is a startup time operation */ + bool loading; + /* The next state will be quiescent */ + bool quiescing; + /* The VDO is quiescent, there should be no I/O */ + bool quiescent; + /* Whether an operation is in progress and so no other operation may be started */ + bool operating; +}; + +extern const struct admin_state_code *VDO_ADMIN_STATE_NORMAL_OPERATION; +extern const struct admin_state_code *VDO_ADMIN_STATE_OPERATING; +extern const struct admin_state_code *VDO_ADMIN_STATE_FORMATTING; +extern const struct admin_state_code *VDO_ADMIN_STATE_PRE_LOADING; +extern const struct admin_state_code *VDO_ADMIN_STATE_PRE_LOADED; +extern const struct admin_state_code *VDO_ADMIN_STATE_LOADING; +extern const struct admin_state_code *VDO_ADMIN_STATE_LOADING_FOR_RECOVERY; +extern const struct admin_state_code *VDO_ADMIN_STATE_LOADING_FOR_REBUILD; +extern const struct admin_state_code *VDO_ADMIN_STATE_WAITING_FOR_RECOVERY; +extern const struct admin_state_code *VDO_ADMIN_STATE_NEW; +extern const struct admin_state_code *VDO_ADMIN_STATE_INITIALIZED; +extern const struct admin_state_code *VDO_ADMIN_STATE_RECOVERING; +extern const struct admin_state_code *VDO_ADMIN_STATE_REBUILDING; +extern const struct admin_state_code *VDO_ADMIN_STATE_SAVING; +extern const struct admin_state_code *VDO_ADMIN_STATE_SAVED; +extern const struct admin_state_code *VDO_ADMIN_STATE_SCRUBBING; +extern const struct admin_state_code *VDO_ADMIN_STATE_SAVE_FOR_SCRUBBING; +extern const struct admin_state_code *VDO_ADMIN_STATE_STOPPING; +extern const struct admin_state_code *VDO_ADMIN_STATE_STOPPED; +extern const struct admin_state_code *VDO_ADMIN_STATE_SUSPENDING; +extern const struct admin_state_code *VDO_ADMIN_STATE_SUSPENDED; +extern const struct admin_state_code *VDO_ADMIN_STATE_SUSPENDED_OPERATION; +extern const struct admin_state_code *VDO_ADMIN_STATE_RESUMING; + +struct admin_state { + const struct admin_state_code *current_state; + /* The next administrative state (when the current operation finishes) */ + const struct admin_state_code *next_state; + /* A completion waiting on a state change */ + struct vdo_completion *waiter; + /* Whether an operation is being initiated */ + bool starting; + /* Whether an operation has completed in the initiator */ + bool complete; +}; + +/** + * typedef vdo_admin_initiator_fn - A method to be called once an admin operation may be initiated. + */ +typedef void (*vdo_admin_initiator_fn)(struct admin_state *state); + +static inline const struct admin_state_code * __must_check +vdo_get_admin_state_code(const struct admin_state *state) +{ + return READ_ONCE(state->current_state); +} + +/** + * vdo_set_admin_state_code() - Set the current admin state code. + * + * This function should be used primarily for initialization and by adminState internals. Most uses + * should go through the operation interfaces. + */ +static inline void vdo_set_admin_state_code(struct admin_state *state, + const struct admin_state_code *code) +{ + WRITE_ONCE(state->current_state, code); +} + +static inline bool __must_check vdo_is_state_normal(const struct admin_state *state) +{ + return vdo_get_admin_state_code(state)->normal; +} + +static inline bool __must_check vdo_is_state_suspending(const struct admin_state *state) +{ + return (vdo_get_admin_state_code(state) == VDO_ADMIN_STATE_SUSPENDING); +} + +static inline bool __must_check vdo_is_state_saving(const struct admin_state *state) +{ + return (vdo_get_admin_state_code(state) == VDO_ADMIN_STATE_SAVING); +} + +static inline bool __must_check vdo_is_state_saved(const struct admin_state *state) +{ + return (vdo_get_admin_state_code(state) == VDO_ADMIN_STATE_SAVED); +} + +static inline bool __must_check vdo_is_state_draining(const struct admin_state *state) +{ + return vdo_get_admin_state_code(state)->draining; +} + +static inline bool __must_check vdo_is_state_loading(const struct admin_state *state) +{ + return vdo_get_admin_state_code(state)->loading; +} + +static inline bool __must_check vdo_is_state_resuming(const struct admin_state *state) +{ + return (vdo_get_admin_state_code(state) == VDO_ADMIN_STATE_RESUMING); +} + +static inline bool __must_check vdo_is_state_clean_load(const struct admin_state *state) +{ + const struct admin_state_code *code = vdo_get_admin_state_code(state); + + return ((code == VDO_ADMIN_STATE_FORMATTING) || (code == VDO_ADMIN_STATE_LOADING)); +} + +static inline bool __must_check vdo_is_state_quiescing(const struct admin_state *state) +{ + return vdo_get_admin_state_code(state)->quiescing; +} + +static inline bool __must_check vdo_is_state_quiescent(const struct admin_state *state) +{ + return vdo_get_admin_state_code(state)->quiescent; +} + +bool __must_check vdo_assert_load_operation(const struct admin_state_code *operation, + struct vdo_completion *waiter); + +bool vdo_start_loading(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, vdo_admin_initiator_fn initiator); + +bool vdo_finish_loading(struct admin_state *state); + +bool vdo_finish_loading_with_result(struct admin_state *state, int result); + +bool vdo_start_resuming(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, vdo_admin_initiator_fn initiator); + +bool vdo_finish_resuming(struct admin_state *state); + +bool vdo_finish_resuming_with_result(struct admin_state *state, int result); + +int vdo_resume_if_quiescent(struct admin_state *state); + +bool vdo_start_draining(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, vdo_admin_initiator_fn initiator); + +bool vdo_finish_draining(struct admin_state *state); + +bool vdo_finish_draining_with_result(struct admin_state *state, int result); + +int vdo_start_operation(struct admin_state *state, + const struct admin_state_code *operation); + +int vdo_start_operation_with_waiter(struct admin_state *state, + const struct admin_state_code *operation, + struct vdo_completion *waiter, + vdo_admin_initiator_fn initiator); + +bool vdo_finish_operation(struct admin_state *state, int result); + +#endif /* VDO_ADMIN_STATE_H */ diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c new file mode 100644 index 000000000000..a0a7c1bd634e --- /dev/null +++ b/drivers/md/dm-vdo/block-map.c @@ -0,0 +1,3318 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "block-map.h" + +#include +#include + +#include "errors.h" +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "action-manager.h" +#include "admin-state.h" +#include "completion.h" +#include "constants.h" +#include "data-vio.h" +#include "encodings.h" +#include "io-submitter.h" +#include "physical-zone.h" +#include "recovery-journal.h" +#include "slab-depot.h" +#include "status-codes.h" +#include "types.h" +#include "vdo.h" +#include "vio.h" +#include "wait-queue.h" + +/** + * DOC: Block map eras + * + * The block map era, or maximum age, is used as follows: + * + * Each block map page, when dirty, records the earliest recovery journal block sequence number of + * the changes reflected in that dirty block. Sequence numbers are classified into eras: every + * @maximum_age sequence numbers, we switch to a new era. Block map pages are assigned to eras + * according to the sequence number they record. + * + * In the current (newest) era, block map pages are not written unless there is cache pressure. In + * the next oldest era, each time a new journal block is written 1/@maximum_age of the pages in + * this era are issued for write. In all older eras, pages are issued for write immediately. + */ + +struct page_descriptor { + root_count_t root_index; + height_t height; + page_number_t page_index; + slot_number_t slot; +} __packed; + +union page_key { + struct page_descriptor descriptor; + u64 key; +}; + +struct write_if_not_dirtied_context { + struct block_map_zone *zone; + u8 generation; +}; + +struct block_map_tree_segment { + struct tree_page *levels[VDO_BLOCK_MAP_TREE_HEIGHT]; +}; + +struct block_map_tree { + struct block_map_tree_segment *segments; +}; + +struct forest { + struct block_map *map; + size_t segments; + struct boundary *boundaries; + struct tree_page **pages; + struct block_map_tree trees[]; +}; + +struct cursor_level { + page_number_t page_index; + slot_number_t slot; +}; + +struct cursors; + +struct cursor { + struct vdo_waiter waiter; + struct block_map_tree *tree; + height_t height; + struct cursors *parent; + struct boundary boundary; + struct cursor_level levels[VDO_BLOCK_MAP_TREE_HEIGHT]; + struct pooled_vio *vio; +}; + +struct cursors { + struct block_map_zone *zone; + struct vio_pool *pool; + vdo_entry_callback_fn entry_callback; + struct vdo_completion *completion; + root_count_t active_roots; + struct cursor cursors[]; +}; + +static const physical_block_number_t NO_PAGE = 0xFFFFFFFFFFFFFFFF; + +/* Used to indicate that the page holding the location of a tree root has been "loaded". */ +static const physical_block_number_t VDO_INVALID_PBN = 0xFFFFFFFFFFFFFFFF; + +const struct block_map_entry UNMAPPED_BLOCK_MAP_ENTRY = { + .mapping_state = VDO_MAPPING_STATE_UNMAPPED & 0x0F, + .pbn_high_nibble = 0, + .pbn_low_word = __cpu_to_le32(VDO_ZERO_BLOCK & UINT_MAX), +}; + +#define LOG_INTERVAL 4000 +#define DISPLAY_INTERVAL 100000 + +/* + * For adjusting VDO page cache statistic fields which are only mutated on the logical zone thread. + * Prevents any compiler shenanigans from affecting other threads reading those stats. + */ +#define ADD_ONCE(value, delta) WRITE_ONCE(value, (value) + (delta)) + +static inline bool is_dirty(const struct page_info *info) +{ + return info->state == PS_DIRTY; +} + +static inline bool is_present(const struct page_info *info) +{ + return (info->state == PS_RESIDENT) || (info->state == PS_DIRTY); +} + +static inline bool is_in_flight(const struct page_info *info) +{ + return (info->state == PS_INCOMING) || (info->state == PS_OUTGOING); +} + +static inline bool is_incoming(const struct page_info *info) +{ + return info->state == PS_INCOMING; +} + +static inline bool is_outgoing(const struct page_info *info) +{ + return info->state == PS_OUTGOING; +} + +static inline bool is_valid(const struct page_info *info) +{ + return is_present(info) || is_outgoing(info); +} + +static char *get_page_buffer(struct page_info *info) +{ + struct vdo_page_cache *cache = info->cache; + + return &cache->pages[(info - cache->infos) * VDO_BLOCK_SIZE]; +} + +static inline struct vdo_page_completion *page_completion_from_waiter(struct vdo_waiter *waiter) +{ + struct vdo_page_completion *completion; + + if (waiter == NULL) + return NULL; + + completion = container_of(waiter, struct vdo_page_completion, waiter); + vdo_assert_completion_type(&completion->completion, VDO_PAGE_COMPLETION); + return completion; +} + +/** + * initialize_info() - Initialize all page info structures and put them on the free list. + * + * Return: VDO_SUCCESS or an error. + */ +static int initialize_info(struct vdo_page_cache *cache) +{ + struct page_info *info; + + INIT_LIST_HEAD(&cache->free_list); + for (info = cache->infos; info < cache->infos + cache->page_count; info++) { + int result; + + info->cache = cache; + info->state = PS_FREE; + info->pbn = NO_PAGE; + + result = create_metadata_vio(cache->vdo, VIO_TYPE_BLOCK_MAP, + VIO_PRIORITY_METADATA, info, + get_page_buffer(info), &info->vio); + if (result != VDO_SUCCESS) + return result; + + /* The thread ID should never change. */ + info->vio->completion.callback_thread_id = cache->zone->thread_id; + + INIT_LIST_HEAD(&info->state_entry); + list_add_tail(&info->state_entry, &cache->free_list); + INIT_LIST_HEAD(&info->lru_entry); + } + + return VDO_SUCCESS; +} + +/** + * allocate_cache_components() - Allocate components of the cache which require their own + * allocation. + * @maximum_age: The number of journal blocks before a dirtied page is considered old and must be + * written out. + * + * The caller is responsible for all clean up on errors. + * + * Return: VDO_SUCCESS or an error code. + */ +static int __must_check allocate_cache_components(struct vdo_page_cache *cache) +{ + u64 size = cache->page_count * (u64) VDO_BLOCK_SIZE; + int result; + + result = vdo_allocate(cache->page_count, struct page_info, "page infos", + &cache->infos); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate_memory(size, VDO_BLOCK_SIZE, "cache pages", &cache->pages); + if (result != VDO_SUCCESS) + return result; + + result = vdo_int_map_create(cache->page_count, &cache->page_map); + if (result != VDO_SUCCESS) + return result; + + return initialize_info(cache); +} + +/** + * assert_on_cache_thread() - Assert that a function has been called on the VDO page cache's + * thread. + */ +static inline void assert_on_cache_thread(struct vdo_page_cache *cache, + const char *function_name) +{ + thread_id_t thread_id = vdo_get_callback_thread_id(); + + VDO_ASSERT_LOG_ONLY((thread_id == cache->zone->thread_id), + "%s() must only be called on cache thread %d, not thread %d", + function_name, cache->zone->thread_id, thread_id); +} + +/** assert_io_allowed() - Assert that a page cache may issue I/O. */ +static inline void assert_io_allowed(struct vdo_page_cache *cache) +{ + VDO_ASSERT_LOG_ONLY(!vdo_is_state_quiescent(&cache->zone->state), + "VDO page cache may issue I/O"); +} + +/** report_cache_pressure() - Log and, if enabled, report cache pressure. */ +static void report_cache_pressure(struct vdo_page_cache *cache) +{ + ADD_ONCE(cache->stats.cache_pressure, 1); + if (cache->waiter_count > cache->page_count) { + if ((cache->pressure_report % LOG_INTERVAL) == 0) + vdo_log_info("page cache pressure %u", cache->stats.cache_pressure); + + if (++cache->pressure_report >= DISPLAY_INTERVAL) + cache->pressure_report = 0; + } +} + +/** + * get_page_state_name() - Return the name of a page state. + * + * If the page state is invalid a static string is returned and the invalid state is logged. + * + * Return: A pointer to a static page state name. + */ +static const char * __must_check get_page_state_name(enum vdo_page_buffer_state state) +{ + int result; + static const char * const state_names[] = { + "FREE", "INCOMING", "FAILED", "RESIDENT", "DIRTY", "OUTGOING" + }; + + BUILD_BUG_ON(ARRAY_SIZE(state_names) != PAGE_STATE_COUNT); + + result = VDO_ASSERT(state < ARRAY_SIZE(state_names), + "Unknown page_state value %d", state); + if (result != VDO_SUCCESS) + return "[UNKNOWN PAGE STATE]"; + + return state_names[state]; +} + +/** + * update_counter() - Update the counter associated with a given state. + * @info: The page info to count. + * @delta: The delta to apply to the counter. + */ +static void update_counter(struct page_info *info, s32 delta) +{ + struct block_map_statistics *stats = &info->cache->stats; + + switch (info->state) { + case PS_FREE: + ADD_ONCE(stats->free_pages, delta); + return; + + case PS_INCOMING: + ADD_ONCE(stats->incoming_pages, delta); + return; + + case PS_OUTGOING: + ADD_ONCE(stats->outgoing_pages, delta); + return; + + case PS_FAILED: + ADD_ONCE(stats->failed_pages, delta); + return; + + case PS_RESIDENT: + ADD_ONCE(stats->clean_pages, delta); + return; + + case PS_DIRTY: + ADD_ONCE(stats->dirty_pages, delta); + return; + + default: + return; + } +} + +/** update_lru() - Update the lru information for an active page. */ +static void update_lru(struct page_info *info) +{ + if (info->cache->lru_list.prev != &info->lru_entry) + list_move_tail(&info->lru_entry, &info->cache->lru_list); +} + +/** + * set_info_state() - Set the state of a page_info and put it on the right list, adjusting + * counters. + */ +static void set_info_state(struct page_info *info, enum vdo_page_buffer_state new_state) +{ + if (new_state == info->state) + return; + + update_counter(info, -1); + info->state = new_state; + update_counter(info, 1); + + switch (info->state) { + case PS_FREE: + case PS_FAILED: + list_move_tail(&info->state_entry, &info->cache->free_list); + return; + + case PS_OUTGOING: + list_move_tail(&info->state_entry, &info->cache->outgoing_list); + return; + + case PS_DIRTY: + return; + + default: + list_del_init(&info->state_entry); + } +} + +/** set_info_pbn() - Set the pbn for an info, updating the map as needed. */ +static int __must_check set_info_pbn(struct page_info *info, physical_block_number_t pbn) +{ + struct vdo_page_cache *cache = info->cache; + + /* Either the new or the old page number must be NO_PAGE. */ + int result = VDO_ASSERT((pbn == NO_PAGE) || (info->pbn == NO_PAGE), + "Must free a page before reusing it."); + if (result != VDO_SUCCESS) + return result; + + if (info->pbn != NO_PAGE) + vdo_int_map_remove(cache->page_map, info->pbn); + + info->pbn = pbn; + + if (pbn != NO_PAGE) { + result = vdo_int_map_put(cache->page_map, pbn, info, true, NULL); + if (result != VDO_SUCCESS) + return result; + } + return VDO_SUCCESS; +} + +/** reset_page_info() - Reset page info to represent an unallocated page. */ +static int reset_page_info(struct page_info *info) +{ + int result; + + result = VDO_ASSERT(info->busy == 0, "VDO Page must not be busy"); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT(!vdo_waitq_has_waiters(&info->waiting), + "VDO Page must not have waiters"); + if (result != VDO_SUCCESS) + return result; + + result = set_info_pbn(info, NO_PAGE); + set_info_state(info, PS_FREE); + list_del_init(&info->lru_entry); + return result; +} + +/** + * find_free_page() - Find a free page. + * + * Return: A pointer to the page info structure (if found), NULL otherwise. + */ +static struct page_info * __must_check find_free_page(struct vdo_page_cache *cache) +{ + struct page_info *info; + + info = list_first_entry_or_null(&cache->free_list, struct page_info, + state_entry); + if (info != NULL) + list_del_init(&info->state_entry); + + return info; +} + +/** + * find_page() - Find the page info (if any) associated with a given pbn. + * @pbn: The absolute physical block number of the page. + * + * Return: The page info for the page if available, or NULL if not. + */ +static struct page_info * __must_check find_page(struct vdo_page_cache *cache, + physical_block_number_t pbn) +{ + if ((cache->last_found != NULL) && (cache->last_found->pbn == pbn)) + return cache->last_found; + + cache->last_found = vdo_int_map_get(cache->page_map, pbn); + return cache->last_found; +} + +/** + * select_lru_page() - Determine which page is least recently used. + * + * Picks the least recently used from among the non-busy entries at the front of each of the lru + * ring. Since whenever we mark a page busy we also put it to the end of the ring it is unlikely + * that the entries at the front are busy unless the queue is very short, but not impossible. + * + * Return: A pointer to the info structure for a relevant page, or NULL if no such page can be + * found. The page can be dirty or resident. + */ +static struct page_info * __must_check select_lru_page(struct vdo_page_cache *cache) +{ + struct page_info *info; + + list_for_each_entry(info, &cache->lru_list, lru_entry) + if ((info->busy == 0) && !is_in_flight(info)) + return info; + + return NULL; +} + +/* ASYNCHRONOUS INTERFACE BEYOND THIS POINT */ + +/** + * complete_with_page() - Helper to complete the VDO Page Completion request successfully. + * @info: The page info representing the result page. + * @vdo_page_comp: The VDO page completion to complete. + */ +static void complete_with_page(struct page_info *info, + struct vdo_page_completion *vdo_page_comp) +{ + bool available = vdo_page_comp->writable ? is_present(info) : is_valid(info); + + if (!available) { + vdo_log_error_strerror(VDO_BAD_PAGE, + "Requested cache page %llu in state %s is not %s", + (unsigned long long) info->pbn, + get_page_state_name(info->state), + vdo_page_comp->writable ? "present" : "valid"); + vdo_fail_completion(&vdo_page_comp->completion, VDO_BAD_PAGE); + return; + } + + vdo_page_comp->info = info; + vdo_page_comp->ready = true; + vdo_finish_completion(&vdo_page_comp->completion); +} + +/** + * complete_waiter_with_error() - Complete a page completion with an error code. + * @waiter: The page completion, as a waiter. + * @result_ptr: A pointer to the error code. + * + * Implements waiter_callback_fn. + */ +static void complete_waiter_with_error(struct vdo_waiter *waiter, void *result_ptr) +{ + int *result = result_ptr; + + vdo_fail_completion(&page_completion_from_waiter(waiter)->completion, *result); +} + +/** + * complete_waiter_with_page() - Complete a page completion with a page. + * @waiter: The page completion, as a waiter. + * @page_info: The page info to complete with. + * + * Implements waiter_callback_fn. + */ +static void complete_waiter_with_page(struct vdo_waiter *waiter, void *page_info) +{ + complete_with_page(page_info, page_completion_from_waiter(waiter)); +} + +/** + * distribute_page_over_waitq() - Complete a waitq of VDO page completions with a page result. + * + * Upon completion the waitq will be empty. + * + * Return: The number of pages distributed. + */ +static unsigned int distribute_page_over_waitq(struct page_info *info, + struct vdo_wait_queue *waitq) +{ + size_t num_pages; + + update_lru(info); + num_pages = vdo_waitq_num_waiters(waitq); + + /* + * Increment the busy count once for each pending completion so that this page does not + * stop being busy until all completions have been processed. + */ + info->busy += num_pages; + + vdo_waitq_notify_all_waiters(waitq, complete_waiter_with_page, info); + return num_pages; +} + +/** + * set_persistent_error() - Set a persistent error which all requests will receive in the future. + * @context: A string describing what triggered the error. + * + * Once triggered, all enqueued completions will get this error. Any future requests will result in + * this error as well. + */ +static void set_persistent_error(struct vdo_page_cache *cache, const char *context, + int result) +{ + struct page_info *info; + /* If we're already read-only, there's no need to log. */ + struct vdo *vdo = cache->vdo; + + if ((result != VDO_READ_ONLY) && !vdo_is_read_only(vdo)) { + vdo_log_error_strerror(result, "VDO Page Cache persistent error: %s", + context); + vdo_enter_read_only_mode(vdo, result); + } + + assert_on_cache_thread(cache, __func__); + + vdo_waitq_notify_all_waiters(&cache->free_waiters, + complete_waiter_with_error, &result); + cache->waiter_count = 0; + + for (info = cache->infos; info < cache->infos + cache->page_count; info++) { + vdo_waitq_notify_all_waiters(&info->waiting, + complete_waiter_with_error, &result); + } +} + +/** + * validate_completed_page() - Check that a page completion which is being freed to the cache + * referred to a valid page and is in a valid state. + * @writable: Whether a writable page is required. + * + * Return: VDO_SUCCESS if the page was valid, otherwise as error + */ +static int __must_check validate_completed_page(struct vdo_page_completion *completion, + bool writable) +{ + int result; + + result = VDO_ASSERT(completion->ready, "VDO Page completion not ready"); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT(completion->info != NULL, + "VDO Page Completion must be complete"); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT(completion->info->pbn == completion->pbn, + "VDO Page Completion pbn must be consistent"); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT(is_valid(completion->info), + "VDO Page Completion page must be valid"); + if (result != VDO_SUCCESS) + return result; + + if (writable) { + result = VDO_ASSERT(completion->writable, + "VDO Page Completion must be writable"); + if (result != VDO_SUCCESS) + return result; + } + + return VDO_SUCCESS; +} + +static void check_for_drain_complete(struct block_map_zone *zone) +{ + if (vdo_is_state_draining(&zone->state) && + (zone->active_lookups == 0) && + !vdo_waitq_has_waiters(&zone->flush_waiters) && + !is_vio_pool_busy(zone->vio_pool) && + (zone->page_cache.outstanding_reads == 0) && + (zone->page_cache.outstanding_writes == 0)) { + vdo_finish_draining_with_result(&zone->state, + (vdo_is_read_only(zone->block_map->vdo) ? + VDO_READ_ONLY : VDO_SUCCESS)); + } +} + +static void enter_zone_read_only_mode(struct block_map_zone *zone, int result) +{ + vdo_enter_read_only_mode(zone->block_map->vdo, result); + + /* + * We are in read-only mode, so we won't ever write any page out. + * Just take all waiters off the waitq so the zone can drain. + */ + vdo_waitq_init(&zone->flush_waiters); + check_for_drain_complete(zone); +} + +static bool __must_check +validate_completed_page_or_enter_read_only_mode(struct vdo_page_completion *completion, + bool writable) +{ + int result = validate_completed_page(completion, writable); + + if (result == VDO_SUCCESS) + return true; + + enter_zone_read_only_mode(completion->info->cache->zone, result); + return false; +} + +/** + * handle_load_error() - Handle page load errors. + * @completion: The page read vio. + */ +static void handle_load_error(struct vdo_completion *completion) +{ + int result = completion->result; + struct page_info *info = completion->parent; + struct vdo_page_cache *cache = info->cache; + + assert_on_cache_thread(cache, __func__); + vio_record_metadata_io_error(as_vio(completion)); + vdo_enter_read_only_mode(cache->zone->block_map->vdo, result); + ADD_ONCE(cache->stats.failed_reads, 1); + set_info_state(info, PS_FAILED); + vdo_waitq_notify_all_waiters(&info->waiting, complete_waiter_with_error, &result); + reset_page_info(info); + + /* + * Don't decrement until right before calling check_for_drain_complete() to + * ensure that the above work can't cause the page cache to be freed out from under us. + */ + cache->outstanding_reads--; + check_for_drain_complete(cache->zone); +} + +/** + * page_is_loaded() - Callback used when a page has been loaded. + * @completion: The vio which has loaded the page. Its parent is the page_info. + */ +static void page_is_loaded(struct vdo_completion *completion) +{ + struct page_info *info = completion->parent; + struct vdo_page_cache *cache = info->cache; + nonce_t nonce = info->cache->zone->block_map->nonce; + struct block_map_page *page; + enum block_map_page_validity validity; + + assert_on_cache_thread(cache, __func__); + + page = (struct block_map_page *) get_page_buffer(info); + validity = vdo_validate_block_map_page(page, nonce, info->pbn); + if (validity == VDO_BLOCK_MAP_PAGE_BAD) { + physical_block_number_t pbn = vdo_get_block_map_page_pbn(page); + int result = vdo_log_error_strerror(VDO_BAD_PAGE, + "Expected page %llu but got page %llu instead", + (unsigned long long) info->pbn, + (unsigned long long) pbn); + + vdo_continue_completion(completion, result); + return; + } + + if (validity == VDO_BLOCK_MAP_PAGE_INVALID) + vdo_format_block_map_page(page, nonce, info->pbn, false); + + info->recovery_lock = 0; + set_info_state(info, PS_RESIDENT); + distribute_page_over_waitq(info, &info->waiting); + + /* + * Don't decrement until right before calling check_for_drain_complete() to + * ensure that the above work can't cause the page cache to be freed out from under us. + */ + cache->outstanding_reads--; + check_for_drain_complete(cache->zone); +} + +/** + * handle_rebuild_read_error() - Handle a read error during a read-only rebuild. + * @completion: The page load completion. + */ +static void handle_rebuild_read_error(struct vdo_completion *completion) +{ + struct page_info *info = completion->parent; + struct vdo_page_cache *cache = info->cache; + + assert_on_cache_thread(cache, __func__); + + /* + * We are doing a read-only rebuild, so treat this as a successful read + * of an uninitialized page. + */ + vio_record_metadata_io_error(as_vio(completion)); + ADD_ONCE(cache->stats.failed_reads, 1); + memset(get_page_buffer(info), 0, VDO_BLOCK_SIZE); + vdo_reset_completion(completion); + page_is_loaded(completion); +} + +static void load_cache_page_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct page_info *info = vio->completion.parent; + + continue_vio_after_io(vio, page_is_loaded, info->cache->zone->thread_id); +} + +/** + * launch_page_load() - Begin the process of loading a page. + * + * Return: VDO_SUCCESS or an error code. + */ +static int __must_check launch_page_load(struct page_info *info, + physical_block_number_t pbn) +{ + int result; + vdo_action_fn callback; + struct vdo_page_cache *cache = info->cache; + + assert_io_allowed(cache); + + result = set_info_pbn(info, pbn); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT((info->busy == 0), "Page is not busy before loading."); + if (result != VDO_SUCCESS) + return result; + + set_info_state(info, PS_INCOMING); + cache->outstanding_reads++; + ADD_ONCE(cache->stats.pages_loaded, 1); + callback = (cache->rebuilding ? handle_rebuild_read_error : handle_load_error); + vdo_submit_metadata_vio(info->vio, pbn, load_cache_page_endio, + callback, REQ_OP_READ | REQ_PRIO); + return VDO_SUCCESS; +} + +static void write_pages(struct vdo_completion *completion); + +/** handle_flush_error() - Handle errors flushing the layer. */ +static void handle_flush_error(struct vdo_completion *completion) +{ + struct page_info *info = completion->parent; + + vio_record_metadata_io_error(as_vio(completion)); + set_persistent_error(info->cache, "flush failed", completion->result); + write_pages(completion); +} + +static void flush_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct page_info *info = vio->completion.parent; + + continue_vio_after_io(vio, write_pages, info->cache->zone->thread_id); +} + +/** save_pages() - Attempt to save the outgoing pages by first flushing the layer. */ +static void save_pages(struct vdo_page_cache *cache) +{ + struct page_info *info; + struct vio *vio; + + if ((cache->pages_in_flush > 0) || (cache->pages_to_flush == 0)) + return; + + assert_io_allowed(cache); + + info = list_first_entry(&cache->outgoing_list, struct page_info, state_entry); + + cache->pages_in_flush = cache->pages_to_flush; + cache->pages_to_flush = 0; + ADD_ONCE(cache->stats.flush_count, 1); + + vio = info->vio; + + /* + * We must make sure that the recovery journal entries that changed these pages were + * successfully persisted, and thus must issue a flush before each batch of pages is + * written to ensure this. + */ + vdo_submit_flush_vio(vio, flush_endio, handle_flush_error); +} + +/** + * schedule_page_save() - Add a page to the outgoing list of pages waiting to be saved. + * + * Once in the list, a page may not be used until it has been written out. + */ +static void schedule_page_save(struct page_info *info) +{ + if (info->busy > 0) { + info->write_status = WRITE_STATUS_DEFERRED; + return; + } + + info->cache->pages_to_flush++; + info->cache->outstanding_writes++; + set_info_state(info, PS_OUTGOING); +} + +/** + * launch_page_save() - Add a page to outgoing pages waiting to be saved, and then start saving + * pages if another save is not in progress. + */ +static void launch_page_save(struct page_info *info) +{ + schedule_page_save(info); + save_pages(info->cache); +} + +/** + * completion_needs_page() - Determine whether a given vdo_page_completion (as a waiter) is + * requesting a given page number. + * @context: A pointer to the pbn of the desired page. + * + * Implements waiter_match_fn. + * + * Return: true if the page completion is for the desired page number. + */ +static bool completion_needs_page(struct vdo_waiter *waiter, void *context) +{ + physical_block_number_t *pbn = context; + + return (page_completion_from_waiter(waiter)->pbn == *pbn); +} + +/** + * allocate_free_page() - Allocate a free page to the first completion in the waiting queue, and + * any other completions that match it in page number. + */ +static void allocate_free_page(struct page_info *info) +{ + int result; + struct vdo_waiter *oldest_waiter; + physical_block_number_t pbn; + struct vdo_page_cache *cache = info->cache; + + assert_on_cache_thread(cache, __func__); + + if (!vdo_waitq_has_waiters(&cache->free_waiters)) { + if (cache->stats.cache_pressure > 0) { + vdo_log_info("page cache pressure relieved"); + WRITE_ONCE(cache->stats.cache_pressure, 0); + } + + return; + } + + result = reset_page_info(info); + if (result != VDO_SUCCESS) { + set_persistent_error(cache, "cannot reset page info", result); + return; + } + + oldest_waiter = vdo_waitq_get_first_waiter(&cache->free_waiters); + pbn = page_completion_from_waiter(oldest_waiter)->pbn; + + /* + * Remove all entries which match the page number in question and push them onto the page + * info's waitq. + */ + vdo_waitq_dequeue_matching_waiters(&cache->free_waiters, completion_needs_page, + &pbn, &info->waiting); + cache->waiter_count -= vdo_waitq_num_waiters(&info->waiting); + + result = launch_page_load(info, pbn); + if (result != VDO_SUCCESS) { + vdo_waitq_notify_all_waiters(&info->waiting, + complete_waiter_with_error, &result); + } +} + +/** + * discard_a_page() - Begin the process of discarding a page. + * + * If no page is discardable, increments a count of deferred frees so that the next release of a + * page which is no longer busy will kick off another discard cycle. This is an indication that the + * cache is not big enough. + * + * If the selected page is not dirty, immediately allocates the page to the oldest completion + * waiting for a free page. + */ +static void discard_a_page(struct vdo_page_cache *cache) +{ + struct page_info *info = select_lru_page(cache); + + if (info == NULL) { + report_cache_pressure(cache); + return; + } + + if (!is_dirty(info)) { + allocate_free_page(info); + return; + } + + VDO_ASSERT_LOG_ONLY(!is_in_flight(info), + "page selected for discard is not in flight"); + + cache->discard_count++; + info->write_status = WRITE_STATUS_DISCARD; + launch_page_save(info); +} + +/** + * discard_page_for_completion() - Helper used to trigger a discard so that the completion can get + * a different page. + */ +static void discard_page_for_completion(struct vdo_page_completion *vdo_page_comp) +{ + struct vdo_page_cache *cache = vdo_page_comp->cache; + + cache->waiter_count++; + vdo_waitq_enqueue_waiter(&cache->free_waiters, &vdo_page_comp->waiter); + discard_a_page(cache); +} + +/** + * discard_page_if_needed() - Helper used to trigger a discard if the cache needs another free + * page. + * @cache: The page cache. + */ +static void discard_page_if_needed(struct vdo_page_cache *cache) +{ + if (cache->waiter_count > cache->discard_count) + discard_a_page(cache); +} + +/** + * write_has_finished() - Inform the cache that a write has finished (possibly with an error). + * @info: The info structure for the page whose write just completed. + * + * Return: true if the page write was a discard. + */ +static bool write_has_finished(struct page_info *info) +{ + bool was_discard = (info->write_status == WRITE_STATUS_DISCARD); + + assert_on_cache_thread(info->cache, __func__); + info->cache->outstanding_writes--; + + info->write_status = WRITE_STATUS_NORMAL; + return was_discard; +} + +/** + * handle_page_write_error() - Handler for page write errors. + * @completion: The page write vio. + */ +static void handle_page_write_error(struct vdo_completion *completion) +{ + int result = completion->result; + struct page_info *info = completion->parent; + struct vdo_page_cache *cache = info->cache; + + vio_record_metadata_io_error(as_vio(completion)); + + /* If we're already read-only, write failures are to be expected. */ + if (result != VDO_READ_ONLY) { + vdo_log_ratelimit(vdo_log_error, + "failed to write block map page %llu", + (unsigned long long) info->pbn); + } + + set_info_state(info, PS_DIRTY); + ADD_ONCE(cache->stats.failed_writes, 1); + set_persistent_error(cache, "cannot write page", result); + + if (!write_has_finished(info)) + discard_page_if_needed(cache); + + check_for_drain_complete(cache->zone); +} + +static void page_is_written_out(struct vdo_completion *completion); + +static void write_cache_page_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct page_info *info = vio->completion.parent; + + continue_vio_after_io(vio, page_is_written_out, info->cache->zone->thread_id); +} + +/** + * page_is_written_out() - Callback used when a page has been written out. + * @completion: The vio which wrote the page. Its parent is a page_info. + */ +static void page_is_written_out(struct vdo_completion *completion) +{ + bool was_discard, reclaimed; + u32 reclamations; + struct page_info *info = completion->parent; + struct vdo_page_cache *cache = info->cache; + struct block_map_page *page = (struct block_map_page *) get_page_buffer(info); + + if (!page->header.initialized) { + page->header.initialized = true; + vdo_submit_metadata_vio(info->vio, info->pbn, + write_cache_page_endio, + handle_page_write_error, + REQ_OP_WRITE | REQ_PRIO | REQ_PREFLUSH); + return; + } + + /* Handle journal updates and torn write protection. */ + vdo_release_recovery_journal_block_reference(cache->zone->block_map->journal, + info->recovery_lock, + VDO_ZONE_TYPE_LOGICAL, + cache->zone->zone_number); + info->recovery_lock = 0; + was_discard = write_has_finished(info); + reclaimed = (!was_discard || (info->busy > 0) || vdo_waitq_has_waiters(&info->waiting)); + + set_info_state(info, PS_RESIDENT); + + reclamations = distribute_page_over_waitq(info, &info->waiting); + ADD_ONCE(cache->stats.reclaimed, reclamations); + + if (was_discard) + cache->discard_count--; + + if (reclaimed) + discard_page_if_needed(cache); + else + allocate_free_page(info); + + check_for_drain_complete(cache->zone); +} + +/** + * write_pages() - Write the batch of pages which were covered by the layer flush which just + * completed. + * @flush_completion: The flush vio. + * + * This callback is registered in save_pages(). + */ +static void write_pages(struct vdo_completion *flush_completion) +{ + struct vdo_page_cache *cache = ((struct page_info *) flush_completion->parent)->cache; + + /* + * We need to cache these two values on the stack since it is possible for the last + * page info to cause the page cache to get freed. Hence once we launch the last page, + * it may be unsafe to dereference the cache. + */ + bool has_unflushed_pages = (cache->pages_to_flush > 0); + page_count_t pages_in_flush = cache->pages_in_flush; + + cache->pages_in_flush = 0; + while (pages_in_flush-- > 0) { + struct page_info *info = + list_first_entry(&cache->outgoing_list, struct page_info, + state_entry); + + list_del_init(&info->state_entry); + if (vdo_is_read_only(info->cache->vdo)) { + struct vdo_completion *completion = &info->vio->completion; + + vdo_reset_completion(completion); + completion->callback = page_is_written_out; + completion->error_handler = handle_page_write_error; + vdo_fail_completion(completion, VDO_READ_ONLY); + continue; + } + ADD_ONCE(info->cache->stats.pages_saved, 1); + vdo_submit_metadata_vio(info->vio, info->pbn, write_cache_page_endio, + handle_page_write_error, REQ_OP_WRITE | REQ_PRIO); + } + + if (has_unflushed_pages) { + /* + * If there are unflushed pages, the cache can't have been freed, so this call is + * safe. + */ + save_pages(cache); + } +} + +/** + * vdo_release_page_completion() - Release a VDO Page Completion. + * + * The page referenced by this completion (if any) will no longer be held busy by this completion. + * If a page becomes discardable and there are completions awaiting free pages then a new round of + * page discarding is started. + */ +void vdo_release_page_completion(struct vdo_completion *completion) +{ + struct page_info *discard_info = NULL; + struct vdo_page_completion *page_completion = as_vdo_page_completion(completion); + struct vdo_page_cache *cache; + + if (completion->result == VDO_SUCCESS) { + if (!validate_completed_page_or_enter_read_only_mode(page_completion, false)) + return; + + if (--page_completion->info->busy == 0) + discard_info = page_completion->info; + } + + VDO_ASSERT_LOG_ONLY((page_completion->waiter.next_waiter == NULL), + "Page being released after leaving all queues"); + + page_completion->info = NULL; + cache = page_completion->cache; + assert_on_cache_thread(cache, __func__); + + if (discard_info != NULL) { + if (discard_info->write_status == WRITE_STATUS_DEFERRED) { + discard_info->write_status = WRITE_STATUS_NORMAL; + launch_page_save(discard_info); + } + + /* + * if there are excess requests for pages (that have not already started discards) + * we need to discard some page (which may be this one) + */ + discard_page_if_needed(cache); + } +} + +/** + * load_page_for_completion() - Helper function to load a page as described by a VDO Page + * Completion. + */ +static void load_page_for_completion(struct page_info *info, + struct vdo_page_completion *vdo_page_comp) +{ + int result; + + vdo_waitq_enqueue_waiter(&info->waiting, &vdo_page_comp->waiter); + result = launch_page_load(info, vdo_page_comp->pbn); + if (result != VDO_SUCCESS) { + vdo_waitq_notify_all_waiters(&info->waiting, + complete_waiter_with_error, &result); + } +} + +/** + * vdo_get_page() - Initialize a page completion and get a block map page. + * @page_completion: The vdo_page_completion to initialize. + * @zone: The block map zone of the desired page. + * @pbn: The absolute physical block of the desired page. + * @writable: Whether the page can be modified. + * @parent: The object to notify when the fetch is complete. + * @callback: The notification callback. + * @error_handler: The handler for fetch errors. + * @requeue: Whether we must requeue when notifying the parent. + * + * May cause another page to be discarded (potentially writing a dirty page) and the one nominated + * by the completion to be loaded from disk. When the callback is invoked, the page will be + * resident in the cache and marked busy. All callers must call vdo_release_page_completion() + * when they are done with the page to clear the busy mark. + */ +void vdo_get_page(struct vdo_page_completion *page_completion, + struct block_map_zone *zone, physical_block_number_t pbn, + bool writable, void *parent, vdo_action_fn callback, + vdo_action_fn error_handler, bool requeue) +{ + struct vdo_page_cache *cache = &zone->page_cache; + struct vdo_completion *completion = &page_completion->completion; + struct page_info *info; + + assert_on_cache_thread(cache, __func__); + VDO_ASSERT_LOG_ONLY((page_completion->waiter.next_waiter == NULL), + "New page completion was not already on a wait queue"); + + *page_completion = (struct vdo_page_completion) { + .pbn = pbn, + .writable = writable, + .cache = cache, + }; + + vdo_initialize_completion(completion, cache->vdo, VDO_PAGE_COMPLETION); + vdo_prepare_completion(completion, callback, error_handler, + cache->zone->thread_id, parent); + completion->requeue = requeue; + + if (page_completion->writable && vdo_is_read_only(cache->vdo)) { + vdo_fail_completion(completion, VDO_READ_ONLY); + return; + } + + if (page_completion->writable) + ADD_ONCE(cache->stats.write_count, 1); + else + ADD_ONCE(cache->stats.read_count, 1); + + info = find_page(cache, page_completion->pbn); + if (info != NULL) { + /* The page is in the cache already. */ + if ((info->write_status == WRITE_STATUS_DEFERRED) || + is_incoming(info) || + (is_outgoing(info) && page_completion->writable)) { + /* The page is unusable until it has finished I/O. */ + ADD_ONCE(cache->stats.wait_for_page, 1); + vdo_waitq_enqueue_waiter(&info->waiting, &page_completion->waiter); + return; + } + + if (is_valid(info)) { + /* The page is usable. */ + ADD_ONCE(cache->stats.found_in_cache, 1); + if (!is_present(info)) + ADD_ONCE(cache->stats.read_outgoing, 1); + update_lru(info); + info->busy++; + complete_with_page(info, page_completion); + return; + } + + /* Something horrible has gone wrong. */ + VDO_ASSERT_LOG_ONLY(false, "Info found in a usable state."); + } + + /* The page must be fetched. */ + info = find_free_page(cache); + if (info != NULL) { + ADD_ONCE(cache->stats.fetch_required, 1); + load_page_for_completion(info, page_completion); + return; + } + + /* The page must wait for a page to be discarded. */ + ADD_ONCE(cache->stats.discard_required, 1); + discard_page_for_completion(page_completion); +} + +/** + * vdo_request_page_write() - Request that a VDO page be written out as soon as it is not busy. + * @completion: The vdo_page_completion containing the page. + */ +void vdo_request_page_write(struct vdo_completion *completion) +{ + struct page_info *info; + struct vdo_page_completion *vdo_page_comp = as_vdo_page_completion(completion); + + if (!validate_completed_page_or_enter_read_only_mode(vdo_page_comp, true)) + return; + + info = vdo_page_comp->info; + set_info_state(info, PS_DIRTY); + launch_page_save(info); +} + +/** + * vdo_get_cached_page() - Get the block map page from a page completion. + * @completion: A vdo page completion whose callback has been called. + * @page_ptr: A pointer to hold the page + * + * Return: VDO_SUCCESS or an error + */ +int vdo_get_cached_page(struct vdo_completion *completion, + struct block_map_page **page_ptr) +{ + int result; + struct vdo_page_completion *vpc; + + vpc = as_vdo_page_completion(completion); + result = validate_completed_page(vpc, true); + if (result == VDO_SUCCESS) + *page_ptr = (struct block_map_page *) get_page_buffer(vpc->info); + + return result; +} + +/** + * vdo_invalidate_page_cache() - Invalidate all entries in the VDO page cache. + * + * There must not be any dirty pages in the cache. + * + * Return: A success or error code. + */ +int vdo_invalidate_page_cache(struct vdo_page_cache *cache) +{ + struct page_info *info; + + assert_on_cache_thread(cache, __func__); + + /* Make sure we don't throw away any dirty pages. */ + for (info = cache->infos; info < cache->infos + cache->page_count; info++) { + int result = VDO_ASSERT(!is_dirty(info), "cache must have no dirty pages"); + + if (result != VDO_SUCCESS) + return result; + } + + /* Reset the page map by re-allocating it. */ + vdo_int_map_free(vdo_forget(cache->page_map)); + return vdo_int_map_create(cache->page_count, &cache->page_map); +} + +/** + * get_tree_page_by_index() - Get the tree page for a given height and page index. + * + * Return: The requested page. + */ +static struct tree_page * __must_check get_tree_page_by_index(struct forest *forest, + root_count_t root_index, + height_t height, + page_number_t page_index) +{ + page_number_t offset = 0; + size_t segment; + + for (segment = 0; segment < forest->segments; segment++) { + page_number_t border = forest->boundaries[segment].levels[height - 1]; + + if (page_index < border) { + struct block_map_tree *tree = &forest->trees[root_index]; + + return &(tree->segments[segment].levels[height - 1][page_index - offset]); + } + + offset = border; + } + + return NULL; +} + +/* Get the page referred to by the lock's tree slot at its current height. */ +static inline struct tree_page *get_tree_page(const struct block_map_zone *zone, + const struct tree_lock *lock) +{ + return get_tree_page_by_index(zone->block_map->forest, lock->root_index, + lock->height, + lock->tree_slots[lock->height].page_index); +} + +/** vdo_copy_valid_page() - Validate and copy a buffer to a page. */ +bool vdo_copy_valid_page(char *buffer, nonce_t nonce, + physical_block_number_t pbn, + struct block_map_page *page) +{ + struct block_map_page *loaded = (struct block_map_page *) buffer; + enum block_map_page_validity validity = + vdo_validate_block_map_page(loaded, nonce, pbn); + + if (validity == VDO_BLOCK_MAP_PAGE_VALID) { + memcpy(page, loaded, VDO_BLOCK_SIZE); + return true; + } + + if (validity == VDO_BLOCK_MAP_PAGE_BAD) { + vdo_log_error_strerror(VDO_BAD_PAGE, + "Expected page %llu but got page %llu instead", + (unsigned long long) pbn, + (unsigned long long) vdo_get_block_map_page_pbn(loaded)); + } + + return false; +} + +/** + * in_cyclic_range() - Check whether the given value is between the lower and upper bounds, within + * a cyclic range of values from 0 to (modulus - 1). + * @lower: The lowest value to accept. + * @value: The value to check. + * @upper: The highest value to accept. + * @modulus: The size of the cyclic space, no more than 2^15. + * + * The value and both bounds must be smaller than the modulus. + * + * Return: true if the value is in range. + */ +static bool in_cyclic_range(u16 lower, u16 value, u16 upper, u16 modulus) +{ + if (value < lower) + value += modulus; + if (upper < lower) + upper += modulus; + return (value <= upper); +} + +/** + * is_not_older() - Check whether a generation is strictly older than some other generation in the + * context of a zone's current generation range. + * @zone: The zone in which to do the comparison. + * @a: The generation in question. + * @b: The generation to compare to. + * + * Return: true if generation @a is not strictly older than generation @b in the context of @zone + */ +static bool __must_check is_not_older(struct block_map_zone *zone, u8 a, u8 b) +{ + int result; + + result = VDO_ASSERT((in_cyclic_range(zone->oldest_generation, a, zone->generation, 1 << 8) && + in_cyclic_range(zone->oldest_generation, b, zone->generation, 1 << 8)), + "generation(s) %u, %u are out of range [%u, %u]", + a, b, zone->oldest_generation, zone->generation); + if (result != VDO_SUCCESS) { + enter_zone_read_only_mode(zone, result); + return true; + } + + return in_cyclic_range(b, a, zone->generation, 1 << 8); +} + +static void release_generation(struct block_map_zone *zone, u8 generation) +{ + int result; + + result = VDO_ASSERT((zone->dirty_page_counts[generation] > 0), + "dirty page count underflow for generation %u", generation); + if (result != VDO_SUCCESS) { + enter_zone_read_only_mode(zone, result); + return; + } + + zone->dirty_page_counts[generation]--; + while ((zone->dirty_page_counts[zone->oldest_generation] == 0) && + (zone->oldest_generation != zone->generation)) + zone->oldest_generation++; +} + +static void set_generation(struct block_map_zone *zone, struct tree_page *page, + u8 new_generation) +{ + u32 new_count; + int result; + bool decrement_old = vdo_waiter_is_waiting(&page->waiter); + u8 old_generation = page->generation; + + if (decrement_old && (old_generation == new_generation)) + return; + + page->generation = new_generation; + new_count = ++zone->dirty_page_counts[new_generation]; + result = VDO_ASSERT((new_count != 0), "dirty page count overflow for generation %u", + new_generation); + if (result != VDO_SUCCESS) { + enter_zone_read_only_mode(zone, result); + return; + } + + if (decrement_old) + release_generation(zone, old_generation); +} + +static void write_page(struct tree_page *tree_page, struct pooled_vio *vio); + +/* Implements waiter_callback_fn */ +static void write_page_callback(struct vdo_waiter *waiter, void *context) +{ + write_page(container_of(waiter, struct tree_page, waiter), context); +} + +static void acquire_vio(struct vdo_waiter *waiter, struct block_map_zone *zone) +{ + waiter->callback = write_page_callback; + acquire_vio_from_pool(zone->vio_pool, waiter); +} + +/* Return: true if all possible generations were not already active */ +static bool attempt_increment(struct block_map_zone *zone) +{ + u8 generation = zone->generation + 1; + + if (zone->oldest_generation == generation) + return false; + + zone->generation = generation; + return true; +} + +/* Launches a flush if one is not already in progress. */ +static void enqueue_page(struct tree_page *page, struct block_map_zone *zone) +{ + if ((zone->flusher == NULL) && attempt_increment(zone)) { + zone->flusher = page; + acquire_vio(&page->waiter, zone); + return; + } + + vdo_waitq_enqueue_waiter(&zone->flush_waiters, &page->waiter); +} + +static void write_page_if_not_dirtied(struct vdo_waiter *waiter, void *context) +{ + struct tree_page *page = container_of(waiter, struct tree_page, waiter); + struct write_if_not_dirtied_context *write_context = context; + + if (page->generation == write_context->generation) { + acquire_vio(waiter, write_context->zone); + return; + } + + enqueue_page(page, write_context->zone); +} + +static void return_to_pool(struct block_map_zone *zone, struct pooled_vio *vio) +{ + return_vio_to_pool(zone->vio_pool, vio); + check_for_drain_complete(zone); +} + +/* This callback is registered in write_initialized_page(). */ +static void finish_page_write(struct vdo_completion *completion) +{ + bool dirty; + struct vio *vio = as_vio(completion); + struct pooled_vio *pooled = container_of(vio, struct pooled_vio, vio); + struct tree_page *page = completion->parent; + struct block_map_zone *zone = pooled->context; + + vdo_release_recovery_journal_block_reference(zone->block_map->journal, + page->writing_recovery_lock, + VDO_ZONE_TYPE_LOGICAL, + zone->zone_number); + + dirty = (page->writing_generation != page->generation); + release_generation(zone, page->writing_generation); + page->writing = false; + + if (zone->flusher == page) { + struct write_if_not_dirtied_context context = { + .zone = zone, + .generation = page->writing_generation, + }; + + vdo_waitq_notify_all_waiters(&zone->flush_waiters, + write_page_if_not_dirtied, &context); + if (dirty && attempt_increment(zone)) { + write_page(page, pooled); + return; + } + + zone->flusher = NULL; + } + + if (dirty) { + enqueue_page(page, zone); + } else if ((zone->flusher == NULL) && vdo_waitq_has_waiters(&zone->flush_waiters) && + attempt_increment(zone)) { + zone->flusher = container_of(vdo_waitq_dequeue_waiter(&zone->flush_waiters), + struct tree_page, waiter); + write_page(zone->flusher, pooled); + return; + } + + return_to_pool(zone, pooled); +} + +static void handle_write_error(struct vdo_completion *completion) +{ + int result = completion->result; + struct vio *vio = as_vio(completion); + struct pooled_vio *pooled = container_of(vio, struct pooled_vio, vio); + struct block_map_zone *zone = pooled->context; + + vio_record_metadata_io_error(vio); + enter_zone_read_only_mode(zone, result); + return_to_pool(zone, pooled); +} + +static void write_page_endio(struct bio *bio); + +static void write_initialized_page(struct vdo_completion *completion) +{ + struct vio *vio = as_vio(completion); + struct pooled_vio *pooled = container_of(vio, struct pooled_vio, vio); + struct block_map_zone *zone = pooled->context; + struct tree_page *tree_page = completion->parent; + struct block_map_page *page = (struct block_map_page *) vio->data; + blk_opf_t operation = REQ_OP_WRITE | REQ_PRIO; + + /* + * Now that we know the page has been written at least once, mark the copy we are writing + * as initialized. + */ + page->header.initialized = true; + + if (zone->flusher == tree_page) + operation |= REQ_PREFLUSH; + + vdo_submit_metadata_vio(vio, vdo_get_block_map_page_pbn(page), + write_page_endio, handle_write_error, + operation); +} + +static void write_page_endio(struct bio *bio) +{ + struct pooled_vio *vio = bio->bi_private; + struct block_map_zone *zone = vio->context; + struct block_map_page *page = (struct block_map_page *) vio->vio.data; + + continue_vio_after_io(&vio->vio, + (page->header.initialized ? + finish_page_write : write_initialized_page), + zone->thread_id); +} + +static void write_page(struct tree_page *tree_page, struct pooled_vio *vio) +{ + struct vdo_completion *completion = &vio->vio.completion; + struct block_map_zone *zone = vio->context; + struct block_map_page *page = vdo_as_block_map_page(tree_page); + + if ((zone->flusher != tree_page) && + is_not_older(zone, tree_page->generation, zone->generation)) { + /* + * This page was re-dirtied after the last flush was issued, hence we need to do + * another flush. + */ + enqueue_page(tree_page, zone); + return_to_pool(zone, vio); + return; + } + + completion->parent = tree_page; + memcpy(vio->vio.data, tree_page->page_buffer, VDO_BLOCK_SIZE); + completion->callback_thread_id = zone->thread_id; + + tree_page->writing = true; + tree_page->writing_generation = tree_page->generation; + tree_page->writing_recovery_lock = tree_page->recovery_lock; + + /* Clear this now so that we know this page is not on any dirty list. */ + tree_page->recovery_lock = 0; + + /* + * We've already copied the page into the vio which will write it, so if it was not yet + * initialized, the first write will indicate that (for torn write protection). It is now + * safe to mark it as initialized in memory since if the write fails, the in memory state + * will become irrelevant. + */ + if (page->header.initialized) { + write_initialized_page(completion); + return; + } + + page->header.initialized = true; + vdo_submit_metadata_vio(&vio->vio, vdo_get_block_map_page_pbn(page), + write_page_endio, handle_write_error, + REQ_OP_WRITE | REQ_PRIO); +} + +/* Release a lock on a page which was being loaded or allocated. */ +static void release_page_lock(struct data_vio *data_vio, char *what) +{ + struct block_map_zone *zone; + struct tree_lock *lock_holder; + struct tree_lock *lock = &data_vio->tree_lock; + + VDO_ASSERT_LOG_ONLY(lock->locked, + "release of unlocked block map page %s for key %llu in tree %u", + what, (unsigned long long) lock->key, lock->root_index); + + zone = data_vio->logical.zone->block_map_zone; + lock_holder = vdo_int_map_remove(zone->loading_pages, lock->key); + VDO_ASSERT_LOG_ONLY((lock_holder == lock), + "block map page %s mismatch for key %llu in tree %u", + what, (unsigned long long) lock->key, lock->root_index); + lock->locked = false; +} + +static void finish_lookup(struct data_vio *data_vio, int result) +{ + data_vio->tree_lock.height = 0; + + --data_vio->logical.zone->block_map_zone->active_lookups; + + set_data_vio_logical_callback(data_vio, continue_data_vio_with_block_map_slot); + data_vio->vio.completion.error_handler = handle_data_vio_error; + continue_data_vio_with_error(data_vio, result); +} + +static void abort_lookup_for_waiter(struct vdo_waiter *waiter, void *context) +{ + struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); + int result = *((int *) context); + + if (!data_vio->write) { + if (result == VDO_NO_SPACE) + result = VDO_SUCCESS; + } else if (result != VDO_NO_SPACE) { + result = VDO_READ_ONLY; + } + + finish_lookup(data_vio, result); +} + +static void abort_lookup(struct data_vio *data_vio, int result, char *what) +{ + if (result != VDO_NO_SPACE) + enter_zone_read_only_mode(data_vio->logical.zone->block_map_zone, result); + + if (data_vio->tree_lock.locked) { + release_page_lock(data_vio, what); + vdo_waitq_notify_all_waiters(&data_vio->tree_lock.waiters, + abort_lookup_for_waiter, + &result); + } + + finish_lookup(data_vio, result); +} + +static void abort_load(struct data_vio *data_vio, int result) +{ + abort_lookup(data_vio, result, "load"); +} + +static bool __must_check is_invalid_tree_entry(const struct vdo *vdo, + const struct data_location *mapping, + height_t height) +{ + if (!vdo_is_valid_location(mapping) || + vdo_is_state_compressed(mapping->state) || + (vdo_is_mapped_location(mapping) && (mapping->pbn == VDO_ZERO_BLOCK))) + return true; + + /* Roots aren't physical data blocks, so we can't check their PBNs. */ + if (height == VDO_BLOCK_MAP_TREE_HEIGHT) + return false; + + return !vdo_is_physical_data_block(vdo->depot, mapping->pbn); +} + +static void load_block_map_page(struct block_map_zone *zone, struct data_vio *data_vio); +static void allocate_block_map_page(struct block_map_zone *zone, + struct data_vio *data_vio); + +static void continue_with_loaded_page(struct data_vio *data_vio, + struct block_map_page *page) +{ + struct tree_lock *lock = &data_vio->tree_lock; + struct block_map_tree_slot slot = lock->tree_slots[lock->height]; + struct data_location mapping = + vdo_unpack_block_map_entry(&page->entries[slot.block_map_slot.slot]); + + if (is_invalid_tree_entry(vdo_from_data_vio(data_vio), &mapping, lock->height)) { + vdo_log_error_strerror(VDO_BAD_MAPPING, + "Invalid block map tree PBN: %llu with state %u for page index %u at height %u", + (unsigned long long) mapping.pbn, mapping.state, + lock->tree_slots[lock->height - 1].page_index, + lock->height - 1); + abort_load(data_vio, VDO_BAD_MAPPING); + return; + } + + if (!vdo_is_mapped_location(&mapping)) { + /* The page we need is unallocated */ + allocate_block_map_page(data_vio->logical.zone->block_map_zone, + data_vio); + return; + } + + lock->tree_slots[lock->height - 1].block_map_slot.pbn = mapping.pbn; + if (lock->height == 1) { + finish_lookup(data_vio, VDO_SUCCESS); + return; + } + + /* We know what page we need to load next */ + load_block_map_page(data_vio->logical.zone->block_map_zone, data_vio); +} + +static void continue_load_for_waiter(struct vdo_waiter *waiter, void *context) +{ + struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); + + data_vio->tree_lock.height--; + continue_with_loaded_page(data_vio, context); +} + +static void finish_block_map_page_load(struct vdo_completion *completion) +{ + physical_block_number_t pbn; + struct tree_page *tree_page; + struct block_map_page *page; + nonce_t nonce; + struct vio *vio = as_vio(completion); + struct pooled_vio *pooled = vio_as_pooled_vio(vio); + struct data_vio *data_vio = completion->parent; + struct block_map_zone *zone = pooled->context; + struct tree_lock *tree_lock = &data_vio->tree_lock; + + tree_lock->height--; + pbn = tree_lock->tree_slots[tree_lock->height].block_map_slot.pbn; + tree_page = get_tree_page(zone, tree_lock); + page = (struct block_map_page *) tree_page->page_buffer; + nonce = zone->block_map->nonce; + + if (!vdo_copy_valid_page(vio->data, nonce, pbn, page)) + vdo_format_block_map_page(page, nonce, pbn, false); + return_vio_to_pool(zone->vio_pool, pooled); + + /* Release our claim to the load and wake any waiters */ + release_page_lock(data_vio, "load"); + vdo_waitq_notify_all_waiters(&tree_lock->waiters, continue_load_for_waiter, page); + continue_with_loaded_page(data_vio, page); +} + +static void handle_io_error(struct vdo_completion *completion) +{ + int result = completion->result; + struct vio *vio = as_vio(completion); + struct pooled_vio *pooled = container_of(vio, struct pooled_vio, vio); + struct data_vio *data_vio = completion->parent; + struct block_map_zone *zone = pooled->context; + + vio_record_metadata_io_error(vio); + return_vio_to_pool(zone->vio_pool, pooled); + abort_load(data_vio, result); +} + +static void load_page_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct data_vio *data_vio = vio->completion.parent; + + continue_vio_after_io(vio, finish_block_map_page_load, + data_vio->logical.zone->thread_id); +} + +static void load_page(struct vdo_waiter *waiter, void *context) +{ + struct pooled_vio *pooled = context; + struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); + struct tree_lock *lock = &data_vio->tree_lock; + physical_block_number_t pbn = lock->tree_slots[lock->height - 1].block_map_slot.pbn; + + pooled->vio.completion.parent = data_vio; + vdo_submit_metadata_vio(&pooled->vio, pbn, load_page_endio, + handle_io_error, REQ_OP_READ | REQ_PRIO); +} + +/* + * If the page is already locked, queue up to wait for the lock to be released. If the lock is + * acquired, @data_vio->tree_lock.locked will be true. + */ +static int attempt_page_lock(struct block_map_zone *zone, struct data_vio *data_vio) +{ + int result; + struct tree_lock *lock_holder; + struct tree_lock *lock = &data_vio->tree_lock; + height_t height = lock->height; + struct block_map_tree_slot tree_slot = lock->tree_slots[height]; + union page_key key; + + key.descriptor = (struct page_descriptor) { + .root_index = lock->root_index, + .height = height, + .page_index = tree_slot.page_index, + .slot = tree_slot.block_map_slot.slot, + }; + lock->key = key.key; + + result = vdo_int_map_put(zone->loading_pages, lock->key, + lock, false, (void **) &lock_holder); + if (result != VDO_SUCCESS) + return result; + + if (lock_holder == NULL) { + /* We got the lock */ + data_vio->tree_lock.locked = true; + return VDO_SUCCESS; + } + + /* Someone else is loading or allocating the page we need */ + vdo_waitq_enqueue_waiter(&lock_holder->waiters, &data_vio->waiter); + return VDO_SUCCESS; +} + +/* Load a block map tree page from disk, for the next level in the data vio tree lock. */ +static void load_block_map_page(struct block_map_zone *zone, struct data_vio *data_vio) +{ + int result; + + result = attempt_page_lock(zone, data_vio); + if (result != VDO_SUCCESS) { + abort_load(data_vio, result); + return; + } + + if (data_vio->tree_lock.locked) { + data_vio->waiter.callback = load_page; + acquire_vio_from_pool(zone->vio_pool, &data_vio->waiter); + } +} + +static void allocation_failure(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + if (vdo_requeue_completion_if_needed(completion, + data_vio->logical.zone->thread_id)) + return; + + abort_lookup(data_vio, completion->result, "allocation"); +} + +static void continue_allocation_for_waiter(struct vdo_waiter *waiter, void *context) +{ + struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); + struct tree_lock *tree_lock = &data_vio->tree_lock; + physical_block_number_t pbn = *((physical_block_number_t *) context); + + tree_lock->height--; + data_vio->tree_lock.tree_slots[tree_lock->height].block_map_slot.pbn = pbn; + + if (tree_lock->height == 0) { + finish_lookup(data_vio, VDO_SUCCESS); + return; + } + + allocate_block_map_page(data_vio->logical.zone->block_map_zone, data_vio); +} + +/** expire_oldest_list() - Expire the oldest list. */ +static void expire_oldest_list(struct dirty_lists *dirty_lists) +{ + block_count_t i = dirty_lists->offset++; + + dirty_lists->oldest_period++; + if (!list_empty(&dirty_lists->eras[i][VDO_TREE_PAGE])) { + list_splice_tail_init(&dirty_lists->eras[i][VDO_TREE_PAGE], + &dirty_lists->expired[VDO_TREE_PAGE]); + } + + if (!list_empty(&dirty_lists->eras[i][VDO_CACHE_PAGE])) { + list_splice_tail_init(&dirty_lists->eras[i][VDO_CACHE_PAGE], + &dirty_lists->expired[VDO_CACHE_PAGE]); + } + + if (dirty_lists->offset == dirty_lists->maximum_age) + dirty_lists->offset = 0; +} + + +/** update_period() - Update the dirty_lists period if necessary. */ +static void update_period(struct dirty_lists *dirty, sequence_number_t period) +{ + while (dirty->next_period <= period) { + if ((dirty->next_period - dirty->oldest_period) == dirty->maximum_age) + expire_oldest_list(dirty); + dirty->next_period++; + } +} + +/** write_expired_elements() - Write out the expired list. */ +static void write_expired_elements(struct block_map_zone *zone) +{ + struct tree_page *page, *ttmp; + struct page_info *info, *ptmp; + struct list_head *expired; + u8 generation = zone->generation; + + expired = &zone->dirty_lists->expired[VDO_TREE_PAGE]; + list_for_each_entry_safe(page, ttmp, expired, entry) { + int result; + + list_del_init(&page->entry); + + result = VDO_ASSERT(!vdo_waiter_is_waiting(&page->waiter), + "Newly expired page not already waiting to write"); + if (result != VDO_SUCCESS) { + enter_zone_read_only_mode(zone, result); + continue; + } + + set_generation(zone, page, generation); + if (!page->writing) + enqueue_page(page, zone); + } + + expired = &zone->dirty_lists->expired[VDO_CACHE_PAGE]; + list_for_each_entry_safe(info, ptmp, expired, state_entry) { + list_del_init(&info->state_entry); + schedule_page_save(info); + } + + save_pages(&zone->page_cache); +} + +/** + * add_to_dirty_lists() - Add an element to the dirty lists. + * @zone: The zone in which we are operating. + * @entry: The list entry of the element to add. + * @type: The type of page. + * @old_period: The period in which the element was previously dirtied, or 0 if it was not dirty. + * @new_period: The period in which the element has now been dirtied, or 0 if it does not hold a + * lock. + */ +static void add_to_dirty_lists(struct block_map_zone *zone, + struct list_head *entry, + enum block_map_page_type type, + sequence_number_t old_period, + sequence_number_t new_period) +{ + struct dirty_lists *dirty_lists = zone->dirty_lists; + + if ((old_period == new_period) || ((old_period != 0) && (old_period < new_period))) + return; + + if (new_period < dirty_lists->oldest_period) { + list_move_tail(entry, &dirty_lists->expired[type]); + } else { + update_period(dirty_lists, new_period); + list_move_tail(entry, + &dirty_lists->eras[new_period % dirty_lists->maximum_age][type]); + } + + write_expired_elements(zone); +} + +/* + * Record the allocation in the tree and wake any waiters now that the write lock has been + * released. + */ +static void finish_block_map_allocation(struct vdo_completion *completion) +{ + physical_block_number_t pbn; + struct tree_page *tree_page; + struct block_map_page *page; + sequence_number_t old_lock; + struct data_vio *data_vio = as_data_vio(completion); + struct block_map_zone *zone = data_vio->logical.zone->block_map_zone; + struct tree_lock *tree_lock = &data_vio->tree_lock; + height_t height = tree_lock->height; + + assert_data_vio_in_logical_zone(data_vio); + + tree_page = get_tree_page(zone, tree_lock); + pbn = tree_lock->tree_slots[height - 1].block_map_slot.pbn; + + /* Record the allocation. */ + page = (struct block_map_page *) tree_page->page_buffer; + old_lock = tree_page->recovery_lock; + vdo_update_block_map_page(page, data_vio, pbn, + VDO_MAPPING_STATE_UNCOMPRESSED, + &tree_page->recovery_lock); + + if (vdo_waiter_is_waiting(&tree_page->waiter)) { + /* This page is waiting to be written out. */ + if (zone->flusher != tree_page) { + /* + * The outstanding flush won't cover the update we just made, + * so mark the page as needing another flush. + */ + set_generation(zone, tree_page, zone->generation); + } + } else { + /* Put the page on a dirty list */ + if (old_lock == 0) + INIT_LIST_HEAD(&tree_page->entry); + add_to_dirty_lists(zone, &tree_page->entry, VDO_TREE_PAGE, + old_lock, tree_page->recovery_lock); + } + + tree_lock->height--; + if (height > 1) { + /* Format the interior node we just allocated (in memory). */ + tree_page = get_tree_page(zone, tree_lock); + vdo_format_block_map_page(tree_page->page_buffer, + zone->block_map->nonce, + pbn, false); + } + + /* Release our claim to the allocation and wake any waiters */ + release_page_lock(data_vio, "allocation"); + vdo_waitq_notify_all_waiters(&tree_lock->waiters, + continue_allocation_for_waiter, &pbn); + if (tree_lock->height == 0) { + finish_lookup(data_vio, VDO_SUCCESS); + return; + } + + allocate_block_map_page(zone, data_vio); +} + +static void release_block_map_write_lock(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_allocated_zone(data_vio); + + release_data_vio_allocation_lock(data_vio, true); + launch_data_vio_logical_callback(data_vio, finish_block_map_allocation); +} + +/* + * Newly allocated block map pages are set to have to MAXIMUM_REFERENCES after they are journaled, + * to prevent deduplication against the block after we release the write lock on it, but before we + * write out the page. + */ +static void set_block_map_page_reference_count(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_allocated_zone(data_vio); + + completion->callback = release_block_map_write_lock; + vdo_modify_reference_count(completion, &data_vio->increment_updater); +} + +static void journal_block_map_allocation(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_journal_zone(data_vio); + + set_data_vio_allocated_zone_callback(data_vio, + set_block_map_page_reference_count); + vdo_add_recovery_journal_entry(completion->vdo->recovery_journal, data_vio); +} + +static void allocate_block(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + struct tree_lock *lock = &data_vio->tree_lock; + physical_block_number_t pbn; + + assert_data_vio_in_allocated_zone(data_vio); + + if (!vdo_allocate_block_in_zone(data_vio)) + return; + + pbn = data_vio->allocation.pbn; + lock->tree_slots[lock->height - 1].block_map_slot.pbn = pbn; + data_vio->increment_updater = (struct reference_updater) { + .operation = VDO_JOURNAL_BLOCK_MAP_REMAPPING, + .increment = true, + .zpbn = { + .pbn = pbn, + .state = VDO_MAPPING_STATE_UNCOMPRESSED, + }, + .lock = data_vio->allocation.lock, + }; + + launch_data_vio_journal_callback(data_vio, journal_block_map_allocation); +} + +static void allocate_block_map_page(struct block_map_zone *zone, + struct data_vio *data_vio) +{ + int result; + + if (!data_vio->write || data_vio->is_discard) { + /* This is a pure read or a discard, so there's nothing left to do here. */ + finish_lookup(data_vio, VDO_SUCCESS); + return; + } + + result = attempt_page_lock(zone, data_vio); + if (result != VDO_SUCCESS) { + abort_lookup(data_vio, result, "allocation"); + return; + } + + if (!data_vio->tree_lock.locked) + return; + + data_vio_allocate_data_block(data_vio, VIO_BLOCK_MAP_WRITE_LOCK, + allocate_block, allocation_failure); +} + +/** + * vdo_find_block_map_slot() - Find the block map slot in which the block map entry for a data_vio + * resides and cache that result in the data_vio. + * + * All ancestors in the tree will be allocated or loaded, as needed. + */ +void vdo_find_block_map_slot(struct data_vio *data_vio) +{ + page_number_t page_index; + struct block_map_tree_slot tree_slot; + struct data_location mapping; + struct block_map_page *page = NULL; + struct tree_lock *lock = &data_vio->tree_lock; + struct block_map_zone *zone = data_vio->logical.zone->block_map_zone; + + zone->active_lookups++; + if (vdo_is_state_draining(&zone->state)) { + finish_lookup(data_vio, VDO_SHUTTING_DOWN); + return; + } + + lock->tree_slots[0].block_map_slot.slot = + data_vio->logical.lbn % VDO_BLOCK_MAP_ENTRIES_PER_PAGE; + page_index = (lock->tree_slots[0].page_index / zone->block_map->root_count); + tree_slot = (struct block_map_tree_slot) { + .page_index = page_index / VDO_BLOCK_MAP_ENTRIES_PER_PAGE, + .block_map_slot = { + .pbn = 0, + .slot = page_index % VDO_BLOCK_MAP_ENTRIES_PER_PAGE, + }, + }; + + for (lock->height = 1; lock->height <= VDO_BLOCK_MAP_TREE_HEIGHT; lock->height++) { + physical_block_number_t pbn; + + lock->tree_slots[lock->height] = tree_slot; + page = (struct block_map_page *) (get_tree_page(zone, lock)->page_buffer); + pbn = vdo_get_block_map_page_pbn(page); + if (pbn != VDO_ZERO_BLOCK) { + lock->tree_slots[lock->height].block_map_slot.pbn = pbn; + break; + } + + /* Calculate the index and slot for the next level. */ + tree_slot.block_map_slot.slot = + tree_slot.page_index % VDO_BLOCK_MAP_ENTRIES_PER_PAGE; + tree_slot.page_index = tree_slot.page_index / VDO_BLOCK_MAP_ENTRIES_PER_PAGE; + } + + /* The page at this height has been allocated and loaded. */ + mapping = vdo_unpack_block_map_entry(&page->entries[tree_slot.block_map_slot.slot]); + if (is_invalid_tree_entry(vdo_from_data_vio(data_vio), &mapping, lock->height)) { + vdo_log_error_strerror(VDO_BAD_MAPPING, + "Invalid block map tree PBN: %llu with state %u for page index %u at height %u", + (unsigned long long) mapping.pbn, mapping.state, + lock->tree_slots[lock->height - 1].page_index, + lock->height - 1); + abort_load(data_vio, VDO_BAD_MAPPING); + return; + } + + if (!vdo_is_mapped_location(&mapping)) { + /* The page we want one level down has not been allocated, so allocate it. */ + allocate_block_map_page(zone, data_vio); + return; + } + + lock->tree_slots[lock->height - 1].block_map_slot.pbn = mapping.pbn; + if (lock->height == 1) { + /* This is the ultimate block map page, so we're done */ + finish_lookup(data_vio, VDO_SUCCESS); + return; + } + + /* We know what page we need to load. */ + load_block_map_page(zone, data_vio); +} + +/* + * Find the PBN of a leaf block map page. This method may only be used after all allocated tree + * pages have been loaded, otherwise, it may give the wrong answer (0). + */ +physical_block_number_t vdo_find_block_map_page_pbn(struct block_map *map, + page_number_t page_number) +{ + struct data_location mapping; + struct tree_page *tree_page; + struct block_map_page *page; + root_count_t root_index = page_number % map->root_count; + page_number_t page_index = page_number / map->root_count; + slot_number_t slot = page_index % VDO_BLOCK_MAP_ENTRIES_PER_PAGE; + + page_index /= VDO_BLOCK_MAP_ENTRIES_PER_PAGE; + + tree_page = get_tree_page_by_index(map->forest, root_index, 1, page_index); + page = (struct block_map_page *) tree_page->page_buffer; + if (!page->header.initialized) + return VDO_ZERO_BLOCK; + + mapping = vdo_unpack_block_map_entry(&page->entries[slot]); + if (!vdo_is_valid_location(&mapping) || vdo_is_state_compressed(mapping.state)) + return VDO_ZERO_BLOCK; + return mapping.pbn; +} + +/* + * Write a tree page or indicate that it has been re-dirtied if it is already being written. This + * method is used when correcting errors in the tree during read-only rebuild. + */ +void vdo_write_tree_page(struct tree_page *page, struct block_map_zone *zone) +{ + bool waiting = vdo_waiter_is_waiting(&page->waiter); + + if (waiting && (zone->flusher == page)) + return; + + set_generation(zone, page, zone->generation); + if (waiting || page->writing) + return; + + enqueue_page(page, zone); +} + +static int make_segment(struct forest *old_forest, block_count_t new_pages, + struct boundary *new_boundary, struct forest *forest) +{ + size_t index = (old_forest == NULL) ? 0 : old_forest->segments; + struct tree_page *page_ptr; + page_count_t segment_sizes[VDO_BLOCK_MAP_TREE_HEIGHT]; + height_t height; + root_count_t root; + int result; + + forest->segments = index + 1; + + result = vdo_allocate(forest->segments, struct boundary, + "forest boundary array", &forest->boundaries); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate(forest->segments, struct tree_page *, + "forest page pointers", &forest->pages); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate(new_pages, struct tree_page, + "new forest pages", &forest->pages[index]); + if (result != VDO_SUCCESS) + return result; + + if (index > 0) { + memcpy(forest->boundaries, old_forest->boundaries, + index * sizeof(struct boundary)); + memcpy(forest->pages, old_forest->pages, + index * sizeof(struct tree_page *)); + } + + memcpy(&(forest->boundaries[index]), new_boundary, sizeof(struct boundary)); + + for (height = 0; height < VDO_BLOCK_MAP_TREE_HEIGHT; height++) { + segment_sizes[height] = new_boundary->levels[height]; + if (index > 0) + segment_sizes[height] -= old_forest->boundaries[index - 1].levels[height]; + } + + page_ptr = forest->pages[index]; + for (root = 0; root < forest->map->root_count; root++) { + struct block_map_tree_segment *segment; + struct block_map_tree *tree = &(forest->trees[root]); + height_t height; + + int result = vdo_allocate(forest->segments, + struct block_map_tree_segment, + "tree root segments", &tree->segments); + if (result != VDO_SUCCESS) + return result; + + if (index > 0) { + memcpy(tree->segments, old_forest->trees[root].segments, + index * sizeof(struct block_map_tree_segment)); + } + + segment = &(tree->segments[index]); + for (height = 0; height < VDO_BLOCK_MAP_TREE_HEIGHT; height++) { + if (segment_sizes[height] == 0) + continue; + + segment->levels[height] = page_ptr; + if (height == (VDO_BLOCK_MAP_TREE_HEIGHT - 1)) { + /* Record the root. */ + struct block_map_page *page = + vdo_format_block_map_page(page_ptr->page_buffer, + forest->map->nonce, + VDO_INVALID_PBN, true); + page->entries[0] = + vdo_pack_block_map_entry(forest->map->root_origin + root, + VDO_MAPPING_STATE_UNCOMPRESSED); + } + page_ptr += segment_sizes[height]; + } + } + + return VDO_SUCCESS; +} + +static void deforest(struct forest *forest, size_t first_page_segment) +{ + root_count_t root; + + if (forest->pages != NULL) { + size_t segment; + + for (segment = first_page_segment; segment < forest->segments; segment++) + vdo_free(forest->pages[segment]); + vdo_free(forest->pages); + } + + for (root = 0; root < forest->map->root_count; root++) + vdo_free(forest->trees[root].segments); + + vdo_free(forest->boundaries); + vdo_free(forest); +} + +/** + * make_forest() - Make a collection of trees for a block_map, expanding the existing forest if + * there is one. + * @entries: The number of entries the block map will hold. + * + * Return: VDO_SUCCESS or an error. + */ +static int make_forest(struct block_map *map, block_count_t entries) +{ + struct forest *forest, *old_forest = map->forest; + struct boundary new_boundary, *old_boundary = NULL; + block_count_t new_pages; + int result; + + if (old_forest != NULL) + old_boundary = &(old_forest->boundaries[old_forest->segments - 1]); + + new_pages = vdo_compute_new_forest_pages(map->root_count, old_boundary, + entries, &new_boundary); + if (new_pages == 0) { + map->next_entry_count = entries; + return VDO_SUCCESS; + } + + result = vdo_allocate_extended(struct forest, map->root_count, + struct block_map_tree, __func__, + &forest); + if (result != VDO_SUCCESS) + return result; + + forest->map = map; + result = make_segment(old_forest, new_pages, &new_boundary, forest); + if (result != VDO_SUCCESS) { + deforest(forest, forest->segments - 1); + return result; + } + + map->next_forest = forest; + map->next_entry_count = entries; + return VDO_SUCCESS; +} + +/** + * replace_forest() - Replace a block_map's forest with the already-prepared larger forest. + */ +static void replace_forest(struct block_map *map) +{ + if (map->next_forest != NULL) { + if (map->forest != NULL) + deforest(map->forest, map->forest->segments); + map->forest = vdo_forget(map->next_forest); + } + + map->entry_count = map->next_entry_count; + map->next_entry_count = 0; +} + +/** + * finish_cursor() - Finish the traversal of a single tree. If it was the last cursor, finish the + * traversal. + */ +static void finish_cursor(struct cursor *cursor) +{ + struct cursors *cursors = cursor->parent; + struct vdo_completion *completion = cursors->completion; + + return_vio_to_pool(cursors->pool, vdo_forget(cursor->vio)); + if (--cursors->active_roots > 0) + return; + + vdo_free(cursors); + + vdo_finish_completion(completion); +} + +static void traverse(struct cursor *cursor); + +/** + * continue_traversal() - Continue traversing a block map tree. + * @completion: The VIO doing a read or write. + */ +static void continue_traversal(struct vdo_completion *completion) +{ + vio_record_metadata_io_error(as_vio(completion)); + traverse(completion->parent); +} + +/** + * finish_traversal_load() - Continue traversing a block map tree now that a page has been loaded. + * @completion: The VIO doing the read. + */ +static void finish_traversal_load(struct vdo_completion *completion) +{ + struct cursor *cursor = completion->parent; + height_t height = cursor->height; + struct cursor_level *level = &cursor->levels[height]; + struct tree_page *tree_page = + &(cursor->tree->segments[0].levels[height][level->page_index]); + struct block_map_page *page = (struct block_map_page *) tree_page->page_buffer; + + vdo_copy_valid_page(cursor->vio->vio.data, + cursor->parent->zone->block_map->nonce, + pbn_from_vio_bio(cursor->vio->vio.bio), page); + traverse(cursor); +} + +static void traversal_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct cursor *cursor = vio->completion.parent; + + continue_vio_after_io(vio, finish_traversal_load, + cursor->parent->zone->thread_id); +} + +/** + * traverse() - Traverse a single block map tree. + * + * This is the recursive heart of the traversal process. + */ +static void traverse(struct cursor *cursor) +{ + for (; cursor->height < VDO_BLOCK_MAP_TREE_HEIGHT; cursor->height++) { + height_t height = cursor->height; + struct cursor_level *level = &cursor->levels[height]; + struct tree_page *tree_page = + &(cursor->tree->segments[0].levels[height][level->page_index]); + struct block_map_page *page = (struct block_map_page *) tree_page->page_buffer; + + if (!page->header.initialized) + continue; + + for (; level->slot < VDO_BLOCK_MAP_ENTRIES_PER_PAGE; level->slot++) { + struct cursor_level *next_level; + page_number_t entry_index = + (VDO_BLOCK_MAP_ENTRIES_PER_PAGE * level->page_index) + level->slot; + struct data_location location = + vdo_unpack_block_map_entry(&page->entries[level->slot]); + + if (!vdo_is_valid_location(&location)) { + /* This entry is invalid, so remove it from the page. */ + page->entries[level->slot] = UNMAPPED_BLOCK_MAP_ENTRY; + vdo_write_tree_page(tree_page, cursor->parent->zone); + continue; + } + + if (!vdo_is_mapped_location(&location)) + continue; + + /* Erase mapped entries past the end of the logical space. */ + if (entry_index >= cursor->boundary.levels[height]) { + page->entries[level->slot] = UNMAPPED_BLOCK_MAP_ENTRY; + vdo_write_tree_page(tree_page, cursor->parent->zone); + continue; + } + + if (cursor->height < VDO_BLOCK_MAP_TREE_HEIGHT - 1) { + int result = cursor->parent->entry_callback(location.pbn, + cursor->parent->completion); + if (result != VDO_SUCCESS) { + page->entries[level->slot] = UNMAPPED_BLOCK_MAP_ENTRY; + vdo_write_tree_page(tree_page, cursor->parent->zone); + continue; + } + } + + if (cursor->height == 0) + continue; + + cursor->height--; + next_level = &cursor->levels[cursor->height]; + next_level->page_index = entry_index; + next_level->slot = 0; + level->slot++; + vdo_submit_metadata_vio(&cursor->vio->vio, location.pbn, + traversal_endio, continue_traversal, + REQ_OP_READ | REQ_PRIO); + return; + } + } + + finish_cursor(cursor); +} + +/** + * launch_cursor() - Start traversing a single block map tree now that the cursor has a VIO with + * which to load pages. + * @context: The pooled_vio just acquired. + * + * Implements waiter_callback_fn. + */ +static void launch_cursor(struct vdo_waiter *waiter, void *context) +{ + struct cursor *cursor = container_of(waiter, struct cursor, waiter); + struct pooled_vio *pooled = context; + + cursor->vio = pooled; + pooled->vio.completion.parent = cursor; + pooled->vio.completion.callback_thread_id = cursor->parent->zone->thread_id; + traverse(cursor); +} + +/** + * compute_boundary() - Compute the number of pages used at each level of the given root's tree. + * + * Return: The list of page counts as a boundary structure. + */ +static struct boundary compute_boundary(struct block_map *map, root_count_t root_index) +{ + struct boundary boundary; + height_t height; + page_count_t leaf_pages = vdo_compute_block_map_page_count(map->entry_count); + /* + * Compute the leaf pages for this root. If the number of leaf pages does not distribute + * evenly, we must determine if this root gets an extra page. Extra pages are assigned to + * roots starting from tree 0. + */ + page_count_t last_tree_root = (leaf_pages - 1) % map->root_count; + page_count_t level_pages = leaf_pages / map->root_count; + + if (root_index <= last_tree_root) + level_pages++; + + for (height = 0; height < VDO_BLOCK_MAP_TREE_HEIGHT - 1; height++) { + boundary.levels[height] = level_pages; + level_pages = DIV_ROUND_UP(level_pages, VDO_BLOCK_MAP_ENTRIES_PER_PAGE); + } + + /* The root node always exists, even if the root is otherwise unused. */ + boundary.levels[VDO_BLOCK_MAP_TREE_HEIGHT - 1] = 1; + + return boundary; +} + +/** + * vdo_traverse_forest() - Walk the entire forest of a block map. + * @callback: A function to call with the pbn of each allocated node in the forest. + * @completion: The completion to notify on each traversed PBN, and when traversal completes. + */ +void vdo_traverse_forest(struct block_map *map, vdo_entry_callback_fn callback, + struct vdo_completion *completion) +{ + root_count_t root; + struct cursors *cursors; + int result; + + result = vdo_allocate_extended(struct cursors, map->root_count, + struct cursor, __func__, &cursors); + if (result != VDO_SUCCESS) { + vdo_fail_completion(completion, result); + return; + } + + cursors->zone = &map->zones[0]; + cursors->pool = cursors->zone->vio_pool; + cursors->entry_callback = callback; + cursors->completion = completion; + cursors->active_roots = map->root_count; + for (root = 0; root < map->root_count; root++) { + struct cursor *cursor = &cursors->cursors[root]; + + *cursor = (struct cursor) { + .tree = &map->forest->trees[root], + .height = VDO_BLOCK_MAP_TREE_HEIGHT - 1, + .parent = cursors, + .boundary = compute_boundary(map, root), + }; + + cursor->waiter.callback = launch_cursor; + acquire_vio_from_pool(cursors->pool, &cursor->waiter); + } +} + +/** + * initialize_block_map_zone() - Initialize the per-zone portions of the block map. + * @maximum_age: The number of journal blocks before a dirtied page is considered old and must be + * written out. + */ +static int __must_check initialize_block_map_zone(struct block_map *map, + zone_count_t zone_number, + page_count_t cache_size, + block_count_t maximum_age) +{ + int result; + block_count_t i; + struct vdo *vdo = map->vdo; + struct block_map_zone *zone = &map->zones[zone_number]; + + BUILD_BUG_ON(sizeof(struct page_descriptor) != sizeof(u64)); + + zone->zone_number = zone_number; + zone->thread_id = vdo->thread_config.logical_threads[zone_number]; + zone->block_map = map; + + result = vdo_allocate_extended(struct dirty_lists, maximum_age, + dirty_era_t, __func__, + &zone->dirty_lists); + if (result != VDO_SUCCESS) + return result; + + zone->dirty_lists->maximum_age = maximum_age; + INIT_LIST_HEAD(&zone->dirty_lists->expired[VDO_TREE_PAGE]); + INIT_LIST_HEAD(&zone->dirty_lists->expired[VDO_CACHE_PAGE]); + + for (i = 0; i < maximum_age; i++) { + INIT_LIST_HEAD(&zone->dirty_lists->eras[i][VDO_TREE_PAGE]); + INIT_LIST_HEAD(&zone->dirty_lists->eras[i][VDO_CACHE_PAGE]); + } + + result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, &zone->loading_pages); + if (result != VDO_SUCCESS) + return result; + + result = make_vio_pool(vdo, BLOCK_MAP_VIO_POOL_SIZE, + zone->thread_id, VIO_TYPE_BLOCK_MAP_INTERIOR, + VIO_PRIORITY_METADATA, zone, &zone->vio_pool); + if (result != VDO_SUCCESS) + return result; + + vdo_set_admin_state_code(&zone->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + + zone->page_cache.zone = zone; + zone->page_cache.vdo = vdo; + zone->page_cache.page_count = cache_size / map->zone_count; + zone->page_cache.stats.free_pages = zone->page_cache.page_count; + + result = allocate_cache_components(&zone->page_cache); + if (result != VDO_SUCCESS) + return result; + + /* initialize empty circular queues */ + INIT_LIST_HEAD(&zone->page_cache.lru_list); + INIT_LIST_HEAD(&zone->page_cache.outgoing_list); + + return VDO_SUCCESS; +} + +/* Implements vdo_zone_thread_getter_fn */ +static thread_id_t get_block_map_zone_thread_id(void *context, zone_count_t zone_number) +{ + struct block_map *map = context; + + return map->zones[zone_number].thread_id; +} + +/* Implements vdo_action_preamble_fn */ +static void prepare_for_era_advance(void *context, struct vdo_completion *parent) +{ + struct block_map *map = context; + + map->current_era_point = map->pending_era_point; + vdo_finish_completion(parent); +} + +/* Implements vdo_zone_action_fn */ +static void advance_block_map_zone_era(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct block_map *map = context; + struct block_map_zone *zone = &map->zones[zone_number]; + + update_period(zone->dirty_lists, map->current_era_point); + write_expired_elements(zone); + vdo_finish_completion(parent); +} + +/* + * Schedule an era advance if necessary. This method should not be called directly. Rather, call + * vdo_schedule_default_action() on the block map's action manager. + * + * Implements vdo_action_scheduler_fn. + */ +static bool schedule_era_advance(void *context) +{ + struct block_map *map = context; + + if (map->current_era_point == map->pending_era_point) + return false; + + return vdo_schedule_action(map->action_manager, prepare_for_era_advance, + advance_block_map_zone_era, NULL, NULL); +} + +static void uninitialize_block_map_zone(struct block_map_zone *zone) +{ + struct vdo_page_cache *cache = &zone->page_cache; + + vdo_free(vdo_forget(zone->dirty_lists)); + free_vio_pool(vdo_forget(zone->vio_pool)); + vdo_int_map_free(vdo_forget(zone->loading_pages)); + if (cache->infos != NULL) { + struct page_info *info; + + for (info = cache->infos; info < cache->infos + cache->page_count; info++) + free_vio(vdo_forget(info->vio)); + } + + vdo_int_map_free(vdo_forget(cache->page_map)); + vdo_free(vdo_forget(cache->infos)); + vdo_free(vdo_forget(cache->pages)); +} + +void vdo_free_block_map(struct block_map *map) +{ + zone_count_t zone; + + if (map == NULL) + return; + + for (zone = 0; zone < map->zone_count; zone++) + uninitialize_block_map_zone(&map->zones[zone]); + + vdo_abandon_block_map_growth(map); + if (map->forest != NULL) + deforest(vdo_forget(map->forest), 0); + vdo_free(vdo_forget(map->action_manager)); + vdo_free(map); +} + +/* @journal may be NULL. */ +int vdo_decode_block_map(struct block_map_state_2_0 state, block_count_t logical_blocks, + struct vdo *vdo, struct recovery_journal *journal, + nonce_t nonce, page_count_t cache_size, block_count_t maximum_age, + struct block_map **map_ptr) +{ + struct block_map *map; + int result; + zone_count_t zone = 0; + + BUILD_BUG_ON(VDO_BLOCK_MAP_ENTRIES_PER_PAGE != + ((VDO_BLOCK_SIZE - sizeof(struct block_map_page)) / + sizeof(struct block_map_entry))); + result = VDO_ASSERT(cache_size > 0, "block map cache size is specified"); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate_extended(struct block_map, + vdo->thread_config.logical_zone_count, + struct block_map_zone, __func__, &map); + if (result != VDO_SUCCESS) + return result; + + map->vdo = vdo; + map->root_origin = state.root_origin; + map->root_count = state.root_count; + map->entry_count = logical_blocks; + map->journal = journal; + map->nonce = nonce; + + result = make_forest(map, map->entry_count); + if (result != VDO_SUCCESS) { + vdo_free_block_map(map); + return result; + } + + replace_forest(map); + + map->zone_count = vdo->thread_config.logical_zone_count; + for (zone = 0; zone < map->zone_count; zone++) { + result = initialize_block_map_zone(map, zone, cache_size, maximum_age); + if (result != VDO_SUCCESS) { + vdo_free_block_map(map); + return result; + } + } + + result = vdo_make_action_manager(map->zone_count, get_block_map_zone_thread_id, + vdo_get_recovery_journal_thread_id(journal), + map, schedule_era_advance, vdo, + &map->action_manager); + if (result != VDO_SUCCESS) { + vdo_free_block_map(map); + return result; + } + + *map_ptr = map; + return VDO_SUCCESS; +} + +struct block_map_state_2_0 vdo_record_block_map(const struct block_map *map) +{ + return (struct block_map_state_2_0) { + .flat_page_origin = VDO_BLOCK_MAP_FLAT_PAGE_ORIGIN, + /* This is the flat page count, which has turned out to always be 0. */ + .flat_page_count = 0, + .root_origin = map->root_origin, + .root_count = map->root_count, + }; +} + +/* The block map needs to know the journals' sequence number to initialize the eras. */ +void vdo_initialize_block_map_from_journal(struct block_map *map, + struct recovery_journal *journal) +{ + zone_count_t z = 0; + + map->current_era_point = vdo_get_recovery_journal_current_sequence_number(journal); + map->pending_era_point = map->current_era_point; + + for (z = 0; z < map->zone_count; z++) { + struct dirty_lists *dirty_lists = map->zones[z].dirty_lists; + + VDO_ASSERT_LOG_ONLY(dirty_lists->next_period == 0, "current period not set"); + dirty_lists->oldest_period = map->current_era_point; + dirty_lists->next_period = map->current_era_point + 1; + dirty_lists->offset = map->current_era_point % dirty_lists->maximum_age; + } +} + +/* Compute the logical zone for the LBN of a data vio. */ +zone_count_t vdo_compute_logical_zone(struct data_vio *data_vio) +{ + struct block_map *map = vdo_from_data_vio(data_vio)->block_map; + struct tree_lock *tree_lock = &data_vio->tree_lock; + page_number_t page_number = data_vio->logical.lbn / VDO_BLOCK_MAP_ENTRIES_PER_PAGE; + + tree_lock->tree_slots[0].page_index = page_number; + tree_lock->root_index = page_number % map->root_count; + return (tree_lock->root_index % map->zone_count); +} + +void vdo_advance_block_map_era(struct block_map *map, + sequence_number_t recovery_block_number) +{ + if (map == NULL) + return; + + map->pending_era_point = recovery_block_number; + vdo_schedule_default_action(map->action_manager); +} + +/* Implements vdo_admin_initiator_fn */ +static void initiate_drain(struct admin_state *state) +{ + struct block_map_zone *zone = container_of(state, struct block_map_zone, state); + + VDO_ASSERT_LOG_ONLY((zone->active_lookups == 0), + "%s() called with no active lookups", __func__); + + if (!vdo_is_state_suspending(state)) { + while (zone->dirty_lists->oldest_period < zone->dirty_lists->next_period) + expire_oldest_list(zone->dirty_lists); + write_expired_elements(zone); + } + + check_for_drain_complete(zone); +} + +/* Implements vdo_zone_action_fn. */ +static void drain_zone(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct block_map *map = context; + struct block_map_zone *zone = &map->zones[zone_number]; + + vdo_start_draining(&zone->state, + vdo_get_current_manager_operation(map->action_manager), + parent, initiate_drain); +} + +void vdo_drain_block_map(struct block_map *map, const struct admin_state_code *operation, + struct vdo_completion *parent) +{ + vdo_schedule_operation(map->action_manager, operation, NULL, drain_zone, NULL, + parent); +} + +/* Implements vdo_zone_action_fn. */ +static void resume_block_map_zone(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct block_map *map = context; + struct block_map_zone *zone = &map->zones[zone_number]; + + vdo_fail_completion(parent, vdo_resume_if_quiescent(&zone->state)); +} + +void vdo_resume_block_map(struct block_map *map, struct vdo_completion *parent) +{ + vdo_schedule_operation(map->action_manager, VDO_ADMIN_STATE_RESUMING, + NULL, resume_block_map_zone, NULL, parent); +} + +/* Allocate an expanded collection of trees, for a future growth. */ +int vdo_prepare_to_grow_block_map(struct block_map *map, + block_count_t new_logical_blocks) +{ + if (map->next_entry_count == new_logical_blocks) + return VDO_SUCCESS; + + if (map->next_entry_count > 0) + vdo_abandon_block_map_growth(map); + + if (new_logical_blocks < map->entry_count) { + map->next_entry_count = map->entry_count; + return VDO_SUCCESS; + } + + return make_forest(map, new_logical_blocks); +} + +/* Implements vdo_action_preamble_fn */ +static void grow_forest(void *context, struct vdo_completion *completion) +{ + replace_forest(context); + vdo_finish_completion(completion); +} + +/* Requires vdo_prepare_to_grow_block_map() to have been previously called. */ +void vdo_grow_block_map(struct block_map *map, struct vdo_completion *parent) +{ + vdo_schedule_operation(map->action_manager, + VDO_ADMIN_STATE_SUSPENDED_OPERATION, + grow_forest, NULL, NULL, parent); +} + +void vdo_abandon_block_map_growth(struct block_map *map) +{ + struct forest *forest = vdo_forget(map->next_forest); + + if (forest != NULL) + deforest(forest, forest->segments - 1); + + map->next_entry_count = 0; +} + +/* Release the page completion and then continue the requester. */ +static inline void finish_processing_page(struct vdo_completion *completion, int result) +{ + struct vdo_completion *parent = completion->parent; + + vdo_release_page_completion(completion); + vdo_continue_completion(parent, result); +} + +static void handle_page_error(struct vdo_completion *completion) +{ + finish_processing_page(completion, completion->result); +} + +/* Fetch the mapping page for a block map update, and call the provided handler when fetched. */ +static void fetch_mapping_page(struct data_vio *data_vio, bool modifiable, + vdo_action_fn action) +{ + struct block_map_zone *zone = data_vio->logical.zone->block_map_zone; + + if (vdo_is_state_draining(&zone->state)) { + continue_data_vio_with_error(data_vio, VDO_SHUTTING_DOWN); + return; + } + + vdo_get_page(&data_vio->page_completion, zone, + data_vio->tree_lock.tree_slots[0].block_map_slot.pbn, + modifiable, &data_vio->vio.completion, + action, handle_page_error, false); +} + +/** + * clear_mapped_location() - Clear a data_vio's mapped block location, setting it to be unmapped. + * + * This indicates the block map entry for the logical block is either unmapped or corrupted. + */ +static void clear_mapped_location(struct data_vio *data_vio) +{ + data_vio->mapped = (struct zoned_pbn) { + .state = VDO_MAPPING_STATE_UNMAPPED, + }; +} + +/** + * set_mapped_location() - Decode and validate a block map entry, and set the mapped location of a + * data_vio. + * + * Return: VDO_SUCCESS or VDO_BAD_MAPPING if the map entry is invalid or an error code for any + * other failure + */ +static int __must_check set_mapped_location(struct data_vio *data_vio, + const struct block_map_entry *entry) +{ + /* Unpack the PBN for logging purposes even if the entry is invalid. */ + struct data_location mapped = vdo_unpack_block_map_entry(entry); + + if (vdo_is_valid_location(&mapped)) { + int result; + + result = vdo_get_physical_zone(vdo_from_data_vio(data_vio), + mapped.pbn, &data_vio->mapped.zone); + if (result == VDO_SUCCESS) { + data_vio->mapped.pbn = mapped.pbn; + data_vio->mapped.state = mapped.state; + return VDO_SUCCESS; + } + + /* + * Return all errors not specifically known to be errors from validating the + * location. + */ + if ((result != VDO_OUT_OF_RANGE) && (result != VDO_BAD_MAPPING)) + return result; + } + + /* + * Log the corruption even if we wind up ignoring it for write VIOs, converting all cases + * to VDO_BAD_MAPPING. + */ + vdo_log_error_strerror(VDO_BAD_MAPPING, + "PBN %llu with state %u read from the block map was invalid", + (unsigned long long) mapped.pbn, mapped.state); + + /* + * A read VIO has no option but to report the bad mapping--reading zeros would be hiding + * known data loss. + */ + if (!data_vio->write) + return VDO_BAD_MAPPING; + + /* + * A write VIO only reads this mapping to decref the old block. Treat this as an unmapped + * entry rather than fail the write. + */ + clear_mapped_location(data_vio); + return VDO_SUCCESS; +} + +/* This callback is registered in vdo_get_mapped_block(). */ +static void get_mapping_from_fetched_page(struct vdo_completion *completion) +{ + int result; + struct vdo_page_completion *vpc = as_vdo_page_completion(completion); + const struct block_map_page *page; + const struct block_map_entry *entry; + struct data_vio *data_vio = as_data_vio(completion->parent); + struct block_map_tree_slot *tree_slot; + + if (completion->result != VDO_SUCCESS) { + finish_processing_page(completion, completion->result); + return; + } + + result = validate_completed_page(vpc, false); + if (result != VDO_SUCCESS) { + finish_processing_page(completion, result); + return; + } + + page = (const struct block_map_page *) get_page_buffer(vpc->info); + tree_slot = &data_vio->tree_lock.tree_slots[0]; + entry = &page->entries[tree_slot->block_map_slot.slot]; + + result = set_mapped_location(data_vio, entry); + finish_processing_page(completion, result); +} + +void vdo_update_block_map_page(struct block_map_page *page, struct data_vio *data_vio, + physical_block_number_t pbn, + enum block_mapping_state mapping_state, + sequence_number_t *recovery_lock) +{ + struct block_map_zone *zone = data_vio->logical.zone->block_map_zone; + struct block_map *block_map = zone->block_map; + struct recovery_journal *journal = block_map->journal; + sequence_number_t old_locked, new_locked; + struct tree_lock *tree_lock = &data_vio->tree_lock; + + /* Encode the new mapping. */ + page->entries[tree_lock->tree_slots[tree_lock->height].block_map_slot.slot] = + vdo_pack_block_map_entry(pbn, mapping_state); + + /* Adjust references on the recovery journal blocks. */ + old_locked = *recovery_lock; + new_locked = data_vio->recovery_sequence_number; + + if ((old_locked == 0) || (old_locked > new_locked)) { + vdo_acquire_recovery_journal_block_reference(journal, new_locked, + VDO_ZONE_TYPE_LOGICAL, + zone->zone_number); + + if (old_locked > 0) { + vdo_release_recovery_journal_block_reference(journal, old_locked, + VDO_ZONE_TYPE_LOGICAL, + zone->zone_number); + } + + *recovery_lock = new_locked; + } + + /* + * FIXME: explain this more + * Release the transferred lock from the data_vio. + */ + vdo_release_journal_entry_lock(journal, new_locked); + data_vio->recovery_sequence_number = 0; +} + +static void put_mapping_in_fetched_page(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion->parent); + sequence_number_t old_lock; + struct vdo_page_completion *vpc; + struct page_info *info; + int result; + + if (completion->result != VDO_SUCCESS) { + finish_processing_page(completion, completion->result); + return; + } + + vpc = as_vdo_page_completion(completion); + result = validate_completed_page(vpc, true); + if (result != VDO_SUCCESS) { + finish_processing_page(completion, result); + return; + } + + info = vpc->info; + old_lock = info->recovery_lock; + vdo_update_block_map_page((struct block_map_page *) get_page_buffer(info), + data_vio, data_vio->new_mapped.pbn, + data_vio->new_mapped.state, &info->recovery_lock); + set_info_state(info, PS_DIRTY); + add_to_dirty_lists(info->cache->zone, &info->state_entry, + VDO_CACHE_PAGE, old_lock, info->recovery_lock); + finish_processing_page(completion, VDO_SUCCESS); +} + +/* Read a stored block mapping into a data_vio. */ +void vdo_get_mapped_block(struct data_vio *data_vio) +{ + if (data_vio->tree_lock.tree_slots[0].block_map_slot.pbn == VDO_ZERO_BLOCK) { + /* + * We know that the block map page for this LBN has not been allocated, so the + * block must be unmapped. + */ + clear_mapped_location(data_vio); + continue_data_vio(data_vio); + return; + } + + fetch_mapping_page(data_vio, false, get_mapping_from_fetched_page); +} + +/* Update a stored block mapping to reflect a data_vio's new mapping. */ +void vdo_put_mapped_block(struct data_vio *data_vio) +{ + fetch_mapping_page(data_vio, true, put_mapping_in_fetched_page); +} + +struct block_map_statistics vdo_get_block_map_statistics(struct block_map *map) +{ + zone_count_t zone = 0; + struct block_map_statistics totals; + + memset(&totals, 0, sizeof(struct block_map_statistics)); + for (zone = 0; zone < map->zone_count; zone++) { + const struct block_map_statistics *stats = + &(map->zones[zone].page_cache.stats); + + totals.dirty_pages += READ_ONCE(stats->dirty_pages); + totals.clean_pages += READ_ONCE(stats->clean_pages); + totals.free_pages += READ_ONCE(stats->free_pages); + totals.failed_pages += READ_ONCE(stats->failed_pages); + totals.incoming_pages += READ_ONCE(stats->incoming_pages); + totals.outgoing_pages += READ_ONCE(stats->outgoing_pages); + totals.cache_pressure += READ_ONCE(stats->cache_pressure); + totals.read_count += READ_ONCE(stats->read_count); + totals.write_count += READ_ONCE(stats->write_count); + totals.failed_reads += READ_ONCE(stats->failed_reads); + totals.failed_writes += READ_ONCE(stats->failed_writes); + totals.reclaimed += READ_ONCE(stats->reclaimed); + totals.read_outgoing += READ_ONCE(stats->read_outgoing); + totals.found_in_cache += READ_ONCE(stats->found_in_cache); + totals.discard_required += READ_ONCE(stats->discard_required); + totals.wait_for_page += READ_ONCE(stats->wait_for_page); + totals.fetch_required += READ_ONCE(stats->fetch_required); + totals.pages_loaded += READ_ONCE(stats->pages_loaded); + totals.pages_saved += READ_ONCE(stats->pages_saved); + totals.flush_count += READ_ONCE(stats->flush_count); + } + + return totals; +} diff --git a/drivers/md/dm-vdo/block-map.h b/drivers/md/dm-vdo/block-map.h new file mode 100644 index 000000000000..39a13039e4a3 --- /dev/null +++ b/drivers/md/dm-vdo/block-map.h @@ -0,0 +1,394 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_BLOCK_MAP_H +#define VDO_BLOCK_MAP_H + +#include + +#include "numeric.h" + +#include "admin-state.h" +#include "completion.h" +#include "encodings.h" +#include "int-map.h" +#include "statistics.h" +#include "types.h" +#include "vio.h" +#include "wait-queue.h" + +/* + * The block map is responsible for tracking all the logical to physical mappings of a VDO. It + * consists of a collection of 60 radix trees gradually allocated as logical addresses are used. + * Each tree is assigned to a logical zone such that it is easy to compute which zone must handle + * each logical address. Each logical zone also has a dedicated portion of the leaf page cache. + * + * Each logical zone has a single dedicated queue and thread for performing all updates to the + * radix trees assigned to that zone. The concurrency guarantees of this single-threaded model + * allow the code to omit more fine-grained locking for the block map structures. + * + * Load operations must be performed on the admin thread. Normal operations, such as reading and + * updating mappings, must be performed on the appropriate logical zone thread. Save operations + * must be launched from the same admin thread as the original load operation. + */ + +enum { + BLOCK_MAP_VIO_POOL_SIZE = 64, +}; + +/* + * Generation counter for page references. + */ +typedef u32 vdo_page_generation; + +extern const struct block_map_entry UNMAPPED_BLOCK_MAP_ENTRY; + +/* The VDO Page Cache abstraction. */ +struct vdo_page_cache { + /* the VDO which owns this cache */ + struct vdo *vdo; + /* number of pages in cache */ + page_count_t page_count; + /* number of pages to write in the current batch */ + page_count_t pages_in_batch; + /* Whether the VDO is doing a read-only rebuild */ + bool rebuilding; + + /* array of page information entries */ + struct page_info *infos; + /* raw memory for pages */ + char *pages; + /* cache last found page info */ + struct page_info *last_found; + /* map of page number to info */ + struct int_map *page_map; + /* main LRU list (all infos) */ + struct list_head lru_list; + /* free page list (oldest first) */ + struct list_head free_list; + /* outgoing page list */ + struct list_head outgoing_list; + /* number of read I/O operations pending */ + page_count_t outstanding_reads; + /* number of write I/O operations pending */ + page_count_t outstanding_writes; + /* number of pages covered by the current flush */ + page_count_t pages_in_flush; + /* number of pages waiting to be included in the next flush */ + page_count_t pages_to_flush; + /* number of discards in progress */ + unsigned int discard_count; + /* how many VPCs waiting for free page */ + unsigned int waiter_count; + /* queue of waiters who want a free page */ + struct vdo_wait_queue free_waiters; + /* + * Statistics are only updated on the logical zone thread, but are accessed from other + * threads. + */ + struct block_map_statistics stats; + /* counter for pressure reports */ + u32 pressure_report; + /* the block map zone to which this cache belongs */ + struct block_map_zone *zone; +}; + +/* + * The state of a page buffer. If the page buffer is free no particular page is bound to it, + * otherwise the page buffer is bound to particular page whose absolute pbn is in the pbn field. If + * the page is resident or dirty the page data is stable and may be accessed. Otherwise the page is + * in flight (incoming or outgoing) and its data should not be accessed. + * + * @note Update the static data in get_page_state_name() if you change this enumeration. + */ +enum vdo_page_buffer_state { + /* this page buffer is not being used */ + PS_FREE, + /* this page is being read from store */ + PS_INCOMING, + /* attempt to load this page failed */ + PS_FAILED, + /* this page is valid and un-modified */ + PS_RESIDENT, + /* this page is valid and modified */ + PS_DIRTY, + /* this page is being written and should not be used */ + PS_OUTGOING, + /* not a state */ + PAGE_STATE_COUNT, +} __packed; + +/* + * The write status of page + */ +enum vdo_page_write_status { + WRITE_STATUS_NORMAL, + WRITE_STATUS_DISCARD, + WRITE_STATUS_DEFERRED, +} __packed; + +/* Per-page-slot information. */ +struct page_info { + /* Preallocated page struct vio */ + struct vio *vio; + /* back-link for references */ + struct vdo_page_cache *cache; + /* the pbn of the page */ + physical_block_number_t pbn; + /* page is busy (temporarily locked) */ + u16 busy; + /* the write status the page */ + enum vdo_page_write_status write_status; + /* page state */ + enum vdo_page_buffer_state state; + /* queue of completions awaiting this item */ + struct vdo_wait_queue waiting; + /* state linked list entry */ + struct list_head state_entry; + /* LRU entry */ + struct list_head lru_entry; + /* + * The earliest recovery journal block containing uncommitted updates to the block map page + * associated with this page_info. A reference (lock) is held on that block to prevent it + * from being reaped. When this value changes, the reference on the old value must be + * released and a reference on the new value must be acquired. + */ + sequence_number_t recovery_lock; +}; + +/* + * A completion awaiting a specific page. Also a live reference into the page once completed, until + * freed. + */ +struct vdo_page_completion { + /* The generic completion */ + struct vdo_completion completion; + /* The cache involved */ + struct vdo_page_cache *cache; + /* The waiter for the pending list */ + struct vdo_waiter waiter; + /* The absolute physical block number of the page on disk */ + physical_block_number_t pbn; + /* Whether the page may be modified */ + bool writable; + /* Whether the page is available */ + bool ready; + /* The info structure for the page, only valid when ready */ + struct page_info *info; +}; + +struct forest; + +struct tree_page { + struct vdo_waiter waiter; + + /* Dirty list entry */ + struct list_head entry; + + /* If dirty, the tree zone flush generation in which it was last dirtied. */ + u8 generation; + + /* Whether this page is an interior tree page being written out. */ + bool writing; + + /* If writing, the tree zone flush generation of the copy being written. */ + u8 writing_generation; + + /* + * Sequence number of the earliest recovery journal block containing uncommitted updates to + * this page + */ + sequence_number_t recovery_lock; + + /* The value of recovery_lock when the this page last started writing */ + sequence_number_t writing_recovery_lock; + + char page_buffer[VDO_BLOCK_SIZE]; +}; + +enum block_map_page_type { + VDO_TREE_PAGE, + VDO_CACHE_PAGE, +}; + +typedef struct list_head dirty_era_t[2]; + +struct dirty_lists { + /* The number of periods after which an element will be expired */ + block_count_t maximum_age; + /* The oldest period which has unexpired elements */ + sequence_number_t oldest_period; + /* One more than the current period */ + sequence_number_t next_period; + /* The offset in the array of lists of the oldest period */ + block_count_t offset; + /* Expired pages */ + dirty_era_t expired; + /* The lists of dirty pages */ + dirty_era_t eras[]; +}; + +struct block_map_zone { + zone_count_t zone_number; + thread_id_t thread_id; + struct admin_state state; + struct block_map *block_map; + /* Dirty pages, by era*/ + struct dirty_lists *dirty_lists; + struct vdo_page_cache page_cache; + data_vio_count_t active_lookups; + struct int_map *loading_pages; + struct vio_pool *vio_pool; + /* The tree page which has issued or will be issuing a flush */ + struct tree_page *flusher; + struct vdo_wait_queue flush_waiters; + /* The generation after the most recent flush */ + u8 generation; + u8 oldest_generation; + /* The counts of dirty pages in each generation */ + u32 dirty_page_counts[256]; +}; + +struct block_map { + struct vdo *vdo; + struct action_manager *action_manager; + /* The absolute PBN of the first root of the tree part of the block map */ + physical_block_number_t root_origin; + block_count_t root_count; + + /* The era point we are currently distributing to the zones */ + sequence_number_t current_era_point; + /* The next era point */ + sequence_number_t pending_era_point; + + /* The number of entries in block map */ + block_count_t entry_count; + nonce_t nonce; + struct recovery_journal *journal; + + /* The trees for finding block map pages */ + struct forest *forest; + /* The expanded trees awaiting growth */ + struct forest *next_forest; + /* The number of entries after growth */ + block_count_t next_entry_count; + + zone_count_t zone_count; + struct block_map_zone zones[]; +}; + +/** + * typedef vdo_entry_callback_fn - A function to be called for each allocated PBN when traversing + * the forest. + * @pbn: A PBN of a tree node. + * @completion: The parent completion of the traversal. + * + * Return: VDO_SUCCESS or an error. + */ +typedef int (*vdo_entry_callback_fn)(physical_block_number_t pbn, + struct vdo_completion *completion); + +static inline struct vdo_page_completion *as_vdo_page_completion(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_PAGE_COMPLETION); + return container_of(completion, struct vdo_page_completion, completion); +} + +void vdo_release_page_completion(struct vdo_completion *completion); + +void vdo_get_page(struct vdo_page_completion *page_completion, + struct block_map_zone *zone, physical_block_number_t pbn, + bool writable, void *parent, vdo_action_fn callback, + vdo_action_fn error_handler, bool requeue); + +void vdo_request_page_write(struct vdo_completion *completion); + +int __must_check vdo_get_cached_page(struct vdo_completion *completion, + struct block_map_page **page_ptr); + +int __must_check vdo_invalidate_page_cache(struct vdo_page_cache *cache); + +static inline struct block_map_page * __must_check +vdo_as_block_map_page(struct tree_page *tree_page) +{ + return (struct block_map_page *) tree_page->page_buffer; +} + +bool vdo_copy_valid_page(char *buffer, nonce_t nonce, + physical_block_number_t pbn, + struct block_map_page *page); + +void vdo_find_block_map_slot(struct data_vio *data_vio); + +physical_block_number_t vdo_find_block_map_page_pbn(struct block_map *map, + page_number_t page_number); + +void vdo_write_tree_page(struct tree_page *page, struct block_map_zone *zone); + +void vdo_traverse_forest(struct block_map *map, vdo_entry_callback_fn callback, + struct vdo_completion *completion); + +int __must_check vdo_decode_block_map(struct block_map_state_2_0 state, + block_count_t logical_blocks, struct vdo *vdo, + struct recovery_journal *journal, nonce_t nonce, + page_count_t cache_size, block_count_t maximum_age, + struct block_map **map_ptr); + +void vdo_drain_block_map(struct block_map *map, const struct admin_state_code *operation, + struct vdo_completion *parent); + +void vdo_resume_block_map(struct block_map *map, struct vdo_completion *parent); + +int __must_check vdo_prepare_to_grow_block_map(struct block_map *map, + block_count_t new_logical_blocks); + +void vdo_grow_block_map(struct block_map *map, struct vdo_completion *parent); + +void vdo_abandon_block_map_growth(struct block_map *map); + +void vdo_free_block_map(struct block_map *map); + +struct block_map_state_2_0 __must_check vdo_record_block_map(const struct block_map *map); + +void vdo_initialize_block_map_from_journal(struct block_map *map, + struct recovery_journal *journal); + +zone_count_t vdo_compute_logical_zone(struct data_vio *data_vio); + +void vdo_advance_block_map_era(struct block_map *map, + sequence_number_t recovery_block_number); + +void vdo_update_block_map_page(struct block_map_page *page, struct data_vio *data_vio, + physical_block_number_t pbn, + enum block_mapping_state mapping_state, + sequence_number_t *recovery_lock); + +void vdo_get_mapped_block(struct data_vio *data_vio); + +void vdo_put_mapped_block(struct data_vio *data_vio); + +struct block_map_statistics __must_check vdo_get_block_map_statistics(struct block_map *map); + +/** + * vdo_convert_maximum_age() - Convert the maximum age to reflect the new recovery journal format + * @age: The configured maximum age + * + * Return: The converted age + * + * In the old recovery journal format, each journal block held 311 entries, and every write bio + * made two entries. The old maximum age was half the usable journal length. In the new format, + * each block holds only 217 entries, but each bio only makes one entry. We convert the configured + * age so that the number of writes in a block map era is the same in the old and new formats. This + * keeps the bound on the amount of work required to recover the block map from the recovery + * journal the same across the format change. It also keeps the amortization of block map page + * writes to write bios the same. + */ +static inline block_count_t vdo_convert_maximum_age(block_count_t age) +{ + return DIV_ROUND_UP(age * RECOVERY_JOURNAL_1_ENTRIES_PER_BLOCK, + 2 * RECOVERY_JOURNAL_ENTRIES_PER_BLOCK); +} + +#endif /* VDO_BLOCK_MAP_H */ diff --git a/drivers/md/dm-vdo/completion.c b/drivers/md/dm-vdo/completion.c new file mode 100644 index 000000000000..5ad85334632d --- /dev/null +++ b/drivers/md/dm-vdo/completion.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "completion.h" + +#include + +#include "logger.h" +#include "permassert.h" + +#include "status-codes.h" +#include "types.h" +#include "vio.h" +#include "vdo.h" + +/** + * DOC: vdo completions. + * + * Most of vdo's data structures are lock free, each either belonging to a single "zone," or + * divided into a number of zones whose accesses to the structure do not overlap. During normal + * operation, at most one thread will be operating in any given zone. Each zone has a + * vdo_work_queue which holds vdo_completions that are to be run in that zone. A completion may + * only be enqueued on one queue or operating in a single zone at a time. + * + * At each step of a multi-threaded operation, the completion performing the operation is given a + * callback, error handler, and thread id for the next step. A completion is "run" when it is + * operating on the correct thread (as specified by its callback_thread_id). If the value of its + * "result" field is an error (i.e. not VDO_SUCCESS), the function in its "error_handler" will be + * invoked. If the error_handler is NULL, or there is no error, the function set as its "callback" + * will be invoked. Generally, a completion will not be run directly, but rather will be + * "launched." In this case, it will check whether it is operating on the correct thread. If it is, + * it will run immediately. Otherwise, it will be enqueue on the vdo_work_queue associated with the + * completion's "callback_thread_id". When it is dequeued, it will be on the correct thread, and + * will get run. In some cases, the completion should get queued instead of running immediately, + * even if it is being launched from the correct thread. This is usually in cases where there is a + * long chain of callbacks, all on the same thread, which could overflow the stack. In such cases, + * the completion's "requeue" field should be set to true. Doing so will skip the current thread + * check and simply enqueue the completion. + * + * A completion may be "finished," in which case its "complete" field will be set to true before it + * is next run. It is a bug to attempt to set the result or re-finish a finished completion. + * Because a completion's fields are not safe to examine from any thread other than the one on + * which the completion is currently operating, this field is used only to aid in detecting + * programming errors. It can not be used for cross-thread checking on the status of an operation. + * A completion must be "reset" before it can be reused after it has been finished. Resetting will + * also clear any error from the result field. + **/ + +void vdo_initialize_completion(struct vdo_completion *completion, + struct vdo *vdo, + enum vdo_completion_type type) +{ + memset(completion, 0, sizeof(*completion)); + completion->vdo = vdo; + completion->type = type; + vdo_reset_completion(completion); +} + +static inline void assert_incomplete(struct vdo_completion *completion) +{ + VDO_ASSERT_LOG_ONLY(!completion->complete, "completion is not complete"); +} + +/** + * vdo_set_completion_result() - Set the result of a completion. + * + * Older errors will not be masked. + */ +void vdo_set_completion_result(struct vdo_completion *completion, int result) +{ + assert_incomplete(completion); + if (completion->result == VDO_SUCCESS) + completion->result = result; +} + +/** + * vdo_launch_completion_with_priority() - Run or enqueue a completion. + * @priority: The priority at which to enqueue the completion. + * + * If called on the correct thread (i.e. the one specified in the completion's callback_thread_id + * field) and not marked for requeue, the completion will be run immediately. Otherwise, the + * completion will be enqueued on the specified thread. + */ +void vdo_launch_completion_with_priority(struct vdo_completion *completion, + enum vdo_completion_priority priority) +{ + thread_id_t callback_thread = completion->callback_thread_id; + + if (completion->requeue || (callback_thread != vdo_get_callback_thread_id())) { + vdo_enqueue_completion(completion, priority); + return; + } + + vdo_run_completion(completion); +} + +/** vdo_finish_completion() - Mark a completion as complete and then launch it. */ +void vdo_finish_completion(struct vdo_completion *completion) +{ + assert_incomplete(completion); + completion->complete = true; + if (completion->callback != NULL) + vdo_launch_completion(completion); +} + +void vdo_enqueue_completion(struct vdo_completion *completion, + enum vdo_completion_priority priority) +{ + struct vdo *vdo = completion->vdo; + thread_id_t thread_id = completion->callback_thread_id; + + if (VDO_ASSERT(thread_id < vdo->thread_config.thread_count, + "thread_id %u (completion type %d) is less than thread count %u", + thread_id, completion->type, + vdo->thread_config.thread_count) != VDO_SUCCESS) + BUG(); + + completion->requeue = false; + completion->priority = priority; + completion->my_queue = NULL; + vdo_enqueue_work_queue(vdo->threads[thread_id].queue, completion); +} + +/** + * vdo_requeue_completion_if_needed() - Requeue a completion if not called on the specified thread. + * + * Return: True if the completion was requeued; callers may not access the completion in this case. + */ +bool vdo_requeue_completion_if_needed(struct vdo_completion *completion, + thread_id_t callback_thread_id) +{ + if (vdo_get_callback_thread_id() == callback_thread_id) + return false; + + completion->callback_thread_id = callback_thread_id; + vdo_enqueue_completion(completion, VDO_WORK_Q_DEFAULT_PRIORITY); + return true; +} diff --git a/drivers/md/dm-vdo/completion.h b/drivers/md/dm-vdo/completion.h new file mode 100644 index 000000000000..3407f34ce58c --- /dev/null +++ b/drivers/md/dm-vdo/completion.h @@ -0,0 +1,152 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_COMPLETION_H +#define VDO_COMPLETION_H + +#include "permassert.h" + +#include "status-codes.h" +#include "types.h" + +/** + * vdo_run_completion() - Run a completion's callback or error handler on the current thread. + * + * Context: This function must be called from the correct callback thread. + */ +static inline void vdo_run_completion(struct vdo_completion *completion) +{ + if ((completion->result != VDO_SUCCESS) && (completion->error_handler != NULL)) { + completion->error_handler(completion); + return; + } + + completion->callback(completion); +} + +void vdo_set_completion_result(struct vdo_completion *completion, int result); + +void vdo_initialize_completion(struct vdo_completion *completion, struct vdo *vdo, + enum vdo_completion_type type); + +/** + * vdo_reset_completion() - Reset a completion to a clean state, while keeping the type, vdo and + * parent information. + */ +static inline void vdo_reset_completion(struct vdo_completion *completion) +{ + completion->result = VDO_SUCCESS; + completion->complete = false; +} + +void vdo_launch_completion_with_priority(struct vdo_completion *completion, + enum vdo_completion_priority priority); + +/** + * vdo_launch_completion() - Launch a completion with default priority. + */ +static inline void vdo_launch_completion(struct vdo_completion *completion) +{ + vdo_launch_completion_with_priority(completion, VDO_WORK_Q_DEFAULT_PRIORITY); +} + +/** + * vdo_continue_completion() - Continue processing a completion. + * @result: The current result (will not mask older errors). + * + * Continue processing a completion by setting the current result and calling + * vdo_launch_completion(). + */ +static inline void vdo_continue_completion(struct vdo_completion *completion, int result) +{ + vdo_set_completion_result(completion, result); + vdo_launch_completion(completion); +} + +void vdo_finish_completion(struct vdo_completion *completion); + +/** + * vdo_fail_completion() - Set the result of a completion if it does not already have an error, + * then finish it. + */ +static inline void vdo_fail_completion(struct vdo_completion *completion, int result) +{ + vdo_set_completion_result(completion, result); + vdo_finish_completion(completion); +} + +/** + * vdo_assert_completion_type() - Assert that a completion is of the correct type. + * + * Return: VDO_SUCCESS or an error + */ +static inline int vdo_assert_completion_type(struct vdo_completion *completion, + enum vdo_completion_type expected) +{ + return VDO_ASSERT(expected == completion->type, + "completion type should be %u, not %u", expected, + completion->type); +} + +static inline void vdo_set_completion_callback(struct vdo_completion *completion, + vdo_action_fn callback, + thread_id_t callback_thread_id) +{ + completion->callback = callback; + completion->callback_thread_id = callback_thread_id; +} + +/** + * vdo_launch_completion_callback() - Set the callback for a completion and launch it immediately. + */ +static inline void vdo_launch_completion_callback(struct vdo_completion *completion, + vdo_action_fn callback, + thread_id_t callback_thread_id) +{ + vdo_set_completion_callback(completion, callback, callback_thread_id); + vdo_launch_completion(completion); +} + +/** + * vdo_prepare_completion() - Prepare a completion for launch. + * + * Resets the completion, and then sets its callback, error handler, callback thread, and parent. + */ +static inline void vdo_prepare_completion(struct vdo_completion *completion, + vdo_action_fn callback, + vdo_action_fn error_handler, + thread_id_t callback_thread_id, void *parent) +{ + vdo_reset_completion(completion); + vdo_set_completion_callback(completion, callback, callback_thread_id); + completion->error_handler = error_handler; + completion->parent = parent; +} + +/** + * vdo_prepare_completion_for_requeue() - Prepare a completion for launch ensuring that it will + * always be requeued. + * + * Resets the completion, and then sets its callback, error handler, callback thread, and parent. + */ +static inline void vdo_prepare_completion_for_requeue(struct vdo_completion *completion, + vdo_action_fn callback, + vdo_action_fn error_handler, + thread_id_t callback_thread_id, + void *parent) +{ + vdo_prepare_completion(completion, callback, error_handler, + callback_thread_id, parent); + completion->requeue = true; +} + +void vdo_enqueue_completion(struct vdo_completion *completion, + enum vdo_completion_priority priority); + + +bool vdo_requeue_completion_if_needed(struct vdo_completion *completion, + thread_id_t callback_thread_id); + +#endif /* VDO_COMPLETION_H */ diff --git a/drivers/md/dm-vdo/constants.h b/drivers/md/dm-vdo/constants.h new file mode 100644 index 000000000000..a8c4d6e24b38 --- /dev/null +++ b/drivers/md/dm-vdo/constants.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_CONSTANTS_H +#define VDO_CONSTANTS_H + +#include + +#include "types.h" + +enum { + /* + * The maximum number of contiguous PBNs which will go to a single bio submission queue, + * assuming there is more than one queue. + */ + VDO_BIO_ROTATION_INTERVAL_LIMIT = 1024, + + /* The number of entries on a block map page */ + VDO_BLOCK_MAP_ENTRIES_PER_PAGE = 812, + + /* The origin of the flat portion of the block map */ + VDO_BLOCK_MAP_FLAT_PAGE_ORIGIN = 1, + + /* + * The height of a block map tree. Assuming a root count of 60 and 812 entries per page, + * this is big enough to represent almost 95 PB of logical space. + */ + VDO_BLOCK_MAP_TREE_HEIGHT = 5, + + /* The default number of bio submission queues. */ + DEFAULT_VDO_BIO_SUBMIT_QUEUE_COUNT = 4, + + /* The number of contiguous PBNs to be submitted to a single bio queue. */ + DEFAULT_VDO_BIO_SUBMIT_QUEUE_ROTATE_INTERVAL = 64, + + /* The number of trees in the arboreal block map */ + DEFAULT_VDO_BLOCK_MAP_TREE_ROOT_COUNT = 60, + + /* The default size of the recovery journal, in blocks */ + DEFAULT_VDO_RECOVERY_JOURNAL_SIZE = 32 * 1024, + + /* The default size of each slab journal, in blocks */ + DEFAULT_VDO_SLAB_JOURNAL_SIZE = 224, + + /* Unit test minimum */ + MINIMUM_VDO_SLAB_JOURNAL_BLOCKS = 2, + + /* + * The initial size of lbn_operations and pbn_operations, which is based upon the expected + * maximum number of outstanding VIOs. This value was chosen to make it highly unlikely + * that the maps would need to be resized. + */ + VDO_LOCK_MAP_CAPACITY = 10000, + + /* The maximum number of logical zones */ + MAX_VDO_LOGICAL_ZONES = 60, + + /* The maximum number of physical zones */ + MAX_VDO_PHYSICAL_ZONES = 16, + + /* The base-2 logarithm of the maximum blocks in one slab */ + MAX_VDO_SLAB_BITS = 23, + + /* The maximum number of slabs the slab depot supports */ + MAX_VDO_SLABS = 8192, + + /* + * The maximum number of block map pages to load simultaneously during recovery or rebuild. + */ + MAXIMUM_SIMULTANEOUS_VDO_BLOCK_MAP_RESTORATION_READS = 1024, + + /* The maximum number of entries in the slab summary */ + MAXIMUM_VDO_SLAB_SUMMARY_ENTRIES = MAX_VDO_SLABS * MAX_VDO_PHYSICAL_ZONES, + + /* The maximum number of total threads in a VDO thread configuration. */ + MAXIMUM_VDO_THREADS = 100, + + /* The maximum number of VIOs in the system at once */ + MAXIMUM_VDO_USER_VIOS = 2048, + + /* The only physical block size supported by VDO */ + VDO_BLOCK_SIZE = 4096, + + /* The number of sectors per block */ + VDO_SECTORS_PER_BLOCK = (VDO_BLOCK_SIZE >> SECTOR_SHIFT), + + /* The size of a sector that will not be torn */ + VDO_SECTOR_SIZE = 512, + + /* The physical block number reserved for storing the zero block */ + VDO_ZERO_BLOCK = 0, +}; + +#endif /* VDO_CONSTANTS_H */ diff --git a/drivers/md/dm-vdo/cpu.h b/drivers/md/dm-vdo/cpu.h new file mode 100644 index 000000000000..d6a2615ba657 --- /dev/null +++ b/drivers/md/dm-vdo/cpu.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_CPU_H +#define UDS_CPU_H + +#include + +/** + * uds_prefetch_address() - Minimize cache-miss latency by attempting to move data into a CPU cache + * before it is accessed. + * + * @address: the address to fetch (may be invalid) + * @for_write: must be constant at compile time--false if for reading, true if for writing + */ +static inline void uds_prefetch_address(const void *address, bool for_write) +{ + /* + * for_write won't be a constant if we are compiled with optimization turned off, in which + * case prefetching really doesn't matter. clang can't figure out that if for_write is a + * constant, it can be passed as the second, mandatorily constant argument to prefetch(), + * at least currently on llvm 12. + */ + if (__builtin_constant_p(for_write)) { + if (for_write) + __builtin_prefetch(address, true); + else + __builtin_prefetch(address, false); + } +} + +/** + * uds_prefetch_range() - Minimize cache-miss latency by attempting to move a range of addresses + * into a CPU cache before they are accessed. + * + * @start: the starting address to fetch (may be invalid) + * @size: the number of bytes in the address range + * @for_write: must be constant at compile time--false if for reading, true if for writing + */ +static inline void uds_prefetch_range(const void *start, unsigned int size, + bool for_write) +{ + /* + * Count the number of cache lines to fetch, allowing for the address range to span an + * extra cache line boundary due to address alignment. + */ + const char *address = (const char *) start; + unsigned int offset = ((uintptr_t) address % L1_CACHE_BYTES); + unsigned int cache_lines = (1 + ((size + offset) / L1_CACHE_BYTES)); + + while (cache_lines-- > 0) { + uds_prefetch_address(address, for_write); + address += L1_CACHE_BYTES; + } +} + +#endif /* UDS_CPU_H */ diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c new file mode 100644 index 000000000000..94f6f1ccfb7d --- /dev/null +++ b/drivers/md/dm-vdo/data-vio.c @@ -0,0 +1,2063 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "data-vio.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "murmurhash3.h" +#include "permassert.h" + +#include "block-map.h" +#include "dump.h" +#include "encodings.h" +#include "int-map.h" +#include "io-submitter.h" +#include "logical-zone.h" +#include "packer.h" +#include "recovery-journal.h" +#include "slab-depot.h" +#include "status-codes.h" +#include "types.h" +#include "vdo.h" +#include "vio.h" +#include "wait-queue.h" + +/** + * DOC: Bio flags. + * + * For certain flags set on user bios, if the user bio has not yet been acknowledged, setting those + * flags on our own bio(s) for that request may help underlying layers better fulfill the user + * bio's needs. This constant contains the aggregate of those flags; VDO strips all the other + * flags, as they convey incorrect information. + * + * These flags are always irrelevant if we have already finished the user bio as they are only + * hints on IO importance. If VDO has finished the user bio, any remaining IO done doesn't care how + * important finishing the finished bio was. + * + * Note that bio.c contains the complete list of flags we believe may be set; the following list + * explains the action taken with each of those flags VDO could receive: + * + * * REQ_SYNC: Passed down if the user bio is not yet completed, since it indicates the user bio + * completion is required for further work to be done by the issuer. + * * REQ_META: Passed down if the user bio is not yet completed, since it may mean the lower layer + * treats it as more urgent, similar to REQ_SYNC. + * * REQ_PRIO: Passed down if the user bio is not yet completed, since it indicates the user bio is + * important. + * * REQ_NOMERGE: Set only if the incoming bio was split; irrelevant to VDO IO. + * * REQ_IDLE: Set if the incoming bio had more IO quickly following; VDO's IO pattern doesn't + * match incoming IO, so this flag is incorrect for it. + * * REQ_FUA: Handled separately, and irrelevant to VDO IO otherwise. + * * REQ_RAHEAD: Passed down, as, for reads, it indicates trivial importance. + * * REQ_BACKGROUND: Not passed down, as VIOs are a limited resource and VDO needs them recycled + * ASAP to service heavy load, which is the only place where REQ_BACKGROUND might aid in load + * prioritization. + */ +static blk_opf_t PASSTHROUGH_FLAGS = (REQ_PRIO | REQ_META | REQ_SYNC | REQ_RAHEAD); + +/** + * DOC: + * + * The data_vio_pool maintains the pool of data_vios which a vdo uses to service incoming bios. For + * correctness, and in order to avoid potentially expensive or blocking memory allocations during + * normal operation, the number of concurrently active data_vios is capped. Furthermore, in order + * to avoid starvation of reads and writes, at most 75% of the data_vios may be used for + * discards. The data_vio_pool is responsible for enforcing these limits. Threads submitting bios + * for which a data_vio or discard permit are not available will block until the necessary + * resources are available. The pool is also responsible for distributing resources to blocked + * threads and waking them. Finally, the pool attempts to batch the work of recycling data_vios by + * performing the work of actually assigning resources to blocked threads or placing data_vios back + * into the pool on a single cpu at a time. + * + * The pool contains two "limiters", one for tracking data_vios and one for tracking discard + * permits. The limiters also provide safe cross-thread access to pool statistics without the need + * to take the pool's lock. When a thread submits a bio to a vdo device, it will first attempt to + * get a discard permit if it is a discard, and then to get a data_vio. If the necessary resources + * are available, the incoming bio will be assigned to the acquired data_vio, and it will be + * launched. However, if either of these are unavailable, the arrival time of the bio is recorded + * in the bio's bi_private field, the bio and its submitter are both queued on the appropriate + * limiter and the submitting thread will then put itself to sleep. (note that this mechanism will + * break if jiffies are only 32 bits.) + * + * Whenever a data_vio has completed processing for the bio it was servicing, release_data_vio() + * will be called on it. This function will add the data_vio to a funnel queue, and then check the + * state of the pool. If the pool is not currently processing released data_vios, the pool's + * completion will be enqueued on a cpu queue. This obviates the need for the releasing threads to + * hold the pool's lock, and also batches release work while avoiding starvation of the cpu + * threads. + * + * Whenever the pool's completion is run on a cpu thread, it calls process_release_callback() which + * processes a batch of returned data_vios (currently at most 32) from the pool's funnel queue. For + * each data_vio, it first checks whether that data_vio was processing a discard. If so, and there + * is a blocked bio waiting for a discard permit, that permit is notionally transferred to the + * eldest discard waiter, and that waiter is moved to the end of the list of discard bios waiting + * for a data_vio. If there are no discard waiters, the discard permit is returned to the pool. + * Next, the data_vio is assigned to the oldest blocked bio which either has a discard permit, or + * doesn't need one and relaunched. If neither of these exist, the data_vio is returned to the + * pool. Finally, if any waiting bios were launched, the threads which blocked trying to submit + * them are awakened. + */ + +#define DATA_VIO_RELEASE_BATCH_SIZE 128 + +static const unsigned int VDO_SECTORS_PER_BLOCK_MASK = VDO_SECTORS_PER_BLOCK - 1; +static const u32 COMPRESSION_STATUS_MASK = 0xff; +static const u32 MAY_NOT_COMPRESS_MASK = 0x80000000; + +struct limiter; +typedef void (*assigner_fn)(struct limiter *limiter); + +/* Bookkeeping structure for a single type of resource. */ +struct limiter { + /* The data_vio_pool to which this limiter belongs */ + struct data_vio_pool *pool; + /* The maximum number of data_vios available */ + data_vio_count_t limit; + /* The number of resources in use */ + data_vio_count_t busy; + /* The maximum number of resources ever simultaneously in use */ + data_vio_count_t max_busy; + /* The number of resources to release */ + data_vio_count_t release_count; + /* The number of waiters to wake */ + data_vio_count_t wake_count; + /* The list of waiting bios which are known to process_release_callback() */ + struct bio_list waiters; + /* The list of waiting bios which are not yet known to process_release_callback() */ + struct bio_list new_waiters; + /* The list of waiters which have their permits */ + struct bio_list *permitted_waiters; + /* The function for assigning a resource to a waiter */ + assigner_fn assigner; + /* The queue of blocked threads */ + wait_queue_head_t blocked_threads; + /* The arrival time of the eldest waiter */ + u64 arrival; +}; + +/* + * A data_vio_pool is a collection of preallocated data_vios which may be acquired from any thread, + * and are released in batches. + */ +struct data_vio_pool { + /* Completion for scheduling releases */ + struct vdo_completion completion; + /* The administrative state of the pool */ + struct admin_state state; + /* Lock protecting the pool */ + spinlock_t lock; + /* The main limiter controlling the total data_vios in the pool. */ + struct limiter limiter; + /* The limiter controlling data_vios for discard */ + struct limiter discard_limiter; + /* The list of bios which have discard permits but still need a data_vio */ + struct bio_list permitted_discards; + /* The list of available data_vios */ + struct list_head available; + /* The queue of data_vios waiting to be returned to the pool */ + struct funnel_queue *queue; + /* Whether the pool is processing, or scheduled to process releases */ + atomic_t processing; + /* The data vios in the pool */ + struct data_vio data_vios[]; +}; + +static const char * const ASYNC_OPERATION_NAMES[] = { + "launch", + "acknowledge_write", + "acquire_hash_lock", + "attempt_logical_block_lock", + "lock_duplicate_pbn", + "check_for_duplication", + "cleanup", + "compress_data_vio", + "find_block_map_slot", + "get_mapped_block_for_read", + "get_mapped_block_for_write", + "hash_data_vio", + "journal_remapping", + "vdo_attempt_packing", + "put_mapped_block", + "read_data_vio", + "update_dedupe_index", + "update_reference_counts", + "verify_duplication", + "write_data_vio", +}; + +/* The steps taken cleaning up a VIO, in the order they are performed. */ +enum data_vio_cleanup_stage { + VIO_CLEANUP_START, + VIO_RELEASE_HASH_LOCK = VIO_CLEANUP_START, + VIO_RELEASE_ALLOCATED, + VIO_RELEASE_RECOVERY_LOCKS, + VIO_RELEASE_LOGICAL, + VIO_CLEANUP_DONE +}; + +static inline struct data_vio_pool * __must_check +as_data_vio_pool(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_DATA_VIO_POOL_COMPLETION); + return container_of(completion, struct data_vio_pool, completion); +} + +static inline u64 get_arrival_time(struct bio *bio) +{ + return (u64) bio->bi_private; +} + +/** + * check_for_drain_complete_locked() - Check whether a data_vio_pool has no outstanding data_vios + * or waiters while holding the pool's lock. + */ +static bool check_for_drain_complete_locked(struct data_vio_pool *pool) +{ + if (pool->limiter.busy > 0) + return false; + + VDO_ASSERT_LOG_ONLY((pool->discard_limiter.busy == 0), + "no outstanding discard permits"); + + return (bio_list_empty(&pool->limiter.new_waiters) && + bio_list_empty(&pool->discard_limiter.new_waiters)); +} + +static void initialize_lbn_lock(struct data_vio *data_vio, logical_block_number_t lbn) +{ + struct vdo *vdo = vdo_from_data_vio(data_vio); + zone_count_t zone_number; + struct lbn_lock *lock = &data_vio->logical; + + lock->lbn = lbn; + lock->locked = false; + vdo_waitq_init(&lock->waiters); + zone_number = vdo_compute_logical_zone(data_vio); + lock->zone = &vdo->logical_zones->zones[zone_number]; +} + +static void launch_locked_request(struct data_vio *data_vio) +{ + data_vio->logical.locked = true; + if (data_vio->write) { + struct vdo *vdo = vdo_from_data_vio(data_vio); + + if (vdo_is_read_only(vdo)) { + continue_data_vio_with_error(data_vio, VDO_READ_ONLY); + return; + } + } + + data_vio->last_async_operation = VIO_ASYNC_OP_FIND_BLOCK_MAP_SLOT; + vdo_find_block_map_slot(data_vio); +} + +static void acknowledge_data_vio(struct data_vio *data_vio) +{ + struct vdo *vdo = vdo_from_data_vio(data_vio); + struct bio *bio = data_vio->user_bio; + int error = vdo_status_to_errno(data_vio->vio.completion.result); + + if (bio == NULL) + return; + + VDO_ASSERT_LOG_ONLY((data_vio->remaining_discard <= + (u32) (VDO_BLOCK_SIZE - data_vio->offset)), + "data_vio to acknowledge is not an incomplete discard"); + + data_vio->user_bio = NULL; + vdo_count_bios(&vdo->stats.bios_acknowledged, bio); + if (data_vio->is_partial) + vdo_count_bios(&vdo->stats.bios_acknowledged_partial, bio); + + bio->bi_status = errno_to_blk_status(error); + bio_endio(bio); +} + +static void copy_to_bio(struct bio *bio, char *data_ptr) +{ + struct bio_vec biovec; + struct bvec_iter iter; + + bio_for_each_segment(biovec, bio, iter) { + memcpy_to_bvec(&biovec, data_ptr); + data_ptr += biovec.bv_len; + } +} + +struct data_vio_compression_status get_data_vio_compression_status(struct data_vio *data_vio) +{ + u32 packed = atomic_read(&data_vio->compression.status); + + /* pairs with cmpxchg in set_data_vio_compression_status */ + smp_rmb(); + return (struct data_vio_compression_status) { + .stage = packed & COMPRESSION_STATUS_MASK, + .may_not_compress = ((packed & MAY_NOT_COMPRESS_MASK) != 0), + }; +} + +/** + * pack_status() - Convert a data_vio_compression_status into a u32 which may be stored + * atomically. + * @status: The state to convert. + * + * Return: The compression state packed into a u32. + */ +static u32 __must_check pack_status(struct data_vio_compression_status status) +{ + return status.stage | (status.may_not_compress ? MAY_NOT_COMPRESS_MASK : 0); +} + +/** + * set_data_vio_compression_status() - Set the compression status of a data_vio. + * @state: The expected current status of the data_vio. + * @new_state: The status to set. + * + * Return: true if the new status was set, false if the data_vio's compression status did not + * match the expected state, and so was left unchanged. + */ +static bool __must_check +set_data_vio_compression_status(struct data_vio *data_vio, + struct data_vio_compression_status status, + struct data_vio_compression_status new_status) +{ + u32 actual; + u32 expected = pack_status(status); + u32 replacement = pack_status(new_status); + + /* + * Extra barriers because this was original developed using a CAS operation that implicitly + * had them. + */ + smp_mb__before_atomic(); + actual = atomic_cmpxchg(&data_vio->compression.status, expected, replacement); + /* same as before_atomic */ + smp_mb__after_atomic(); + return (expected == actual); +} + +struct data_vio_compression_status advance_data_vio_compression_stage(struct data_vio *data_vio) +{ + for (;;) { + struct data_vio_compression_status status = + get_data_vio_compression_status(data_vio); + struct data_vio_compression_status new_status = status; + + if (status.stage == DATA_VIO_POST_PACKER) { + /* We're already in the last stage. */ + return status; + } + + if (status.may_not_compress) { + /* + * Compression has been dis-allowed for this VIO, so skip the rest of the + * path and go to the end. + */ + new_status.stage = DATA_VIO_POST_PACKER; + } else { + /* Go to the next state. */ + new_status.stage++; + } + + if (set_data_vio_compression_status(data_vio, status, new_status)) + return new_status; + + /* Another thread changed the status out from under us so try again. */ + } +} + +/** + * cancel_data_vio_compression() - Prevent this data_vio from being compressed or packed. + * + * Return: true if the data_vio is in the packer and the caller was the first caller to cancel it. + */ +bool cancel_data_vio_compression(struct data_vio *data_vio) +{ + struct data_vio_compression_status status, new_status; + + for (;;) { + status = get_data_vio_compression_status(data_vio); + if (status.may_not_compress || (status.stage == DATA_VIO_POST_PACKER)) { + /* This data_vio is already set up to not block in the packer. */ + break; + } + + new_status.stage = status.stage; + new_status.may_not_compress = true; + + if (set_data_vio_compression_status(data_vio, status, new_status)) + break; + } + + return ((status.stage == DATA_VIO_PACKING) && !status.may_not_compress); +} + +/** + * attempt_logical_block_lock() - Attempt to acquire the lock on a logical block. + * @completion: The data_vio for an external data request as a completion. + * + * This is the start of the path for all external requests. It is registered in launch_data_vio(). + */ +static void attempt_logical_block_lock(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + struct lbn_lock *lock = &data_vio->logical; + struct vdo *vdo = vdo_from_data_vio(data_vio); + struct data_vio *lock_holder; + int result; + + assert_data_vio_in_logical_zone(data_vio); + + if (data_vio->logical.lbn >= vdo->states.vdo.config.logical_blocks) { + continue_data_vio_with_error(data_vio, VDO_OUT_OF_RANGE); + return; + } + + result = vdo_int_map_put(lock->zone->lbn_operations, lock->lbn, + data_vio, false, (void **) &lock_holder); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(data_vio, result); + return; + } + + if (lock_holder == NULL) { + /* We got the lock */ + launch_locked_request(data_vio); + return; + } + + result = VDO_ASSERT(lock_holder->logical.locked, "logical block lock held"); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(data_vio, result); + return; + } + + /* + * If the new request is a pure read request (not read-modify-write) and the lock_holder is + * writing and has received an allocation, service the read request immediately by copying + * data from the lock_holder to avoid having to flush the write out of the packer just to + * prevent the read from waiting indefinitely. If the lock_holder does not yet have an + * allocation, prevent it from blocking in the packer and wait on it. This is necessary in + * order to prevent returning data that may not have actually been written. + */ + if (!data_vio->write && READ_ONCE(lock_holder->allocation_succeeded)) { + copy_to_bio(data_vio->user_bio, lock_holder->vio.data + data_vio->offset); + acknowledge_data_vio(data_vio); + complete_data_vio(completion); + return; + } + + data_vio->last_async_operation = VIO_ASYNC_OP_ATTEMPT_LOGICAL_BLOCK_LOCK; + vdo_waitq_enqueue_waiter(&lock_holder->logical.waiters, &data_vio->waiter); + + /* + * Prevent writes and read-modify-writes from blocking indefinitely on lock holders in the + * packer. + */ + if (lock_holder->write && cancel_data_vio_compression(lock_holder)) { + data_vio->compression.lock_holder = lock_holder; + launch_data_vio_packer_callback(data_vio, + vdo_remove_lock_holder_from_packer); + } +} + +/** + * launch_data_vio() - (Re)initialize a data_vio to have a new logical block number, keeping the + * same parent and other state and send it on its way. + */ +static void launch_data_vio(struct data_vio *data_vio, logical_block_number_t lbn) +{ + struct vdo_completion *completion = &data_vio->vio.completion; + + /* + * Clearing the tree lock must happen before initializing the LBN lock, which also adds + * information to the tree lock. + */ + memset(&data_vio->tree_lock, 0, sizeof(data_vio->tree_lock)); + initialize_lbn_lock(data_vio, lbn); + INIT_LIST_HEAD(&data_vio->hash_lock_entry); + INIT_LIST_HEAD(&data_vio->write_entry); + + memset(&data_vio->allocation, 0, sizeof(data_vio->allocation)); + + data_vio->is_duplicate = false; + + memset(&data_vio->record_name, 0, sizeof(data_vio->record_name)); + memset(&data_vio->duplicate, 0, sizeof(data_vio->duplicate)); + vdo_reset_completion(completion); + completion->error_handler = handle_data_vio_error; + set_data_vio_logical_callback(data_vio, attempt_logical_block_lock); + vdo_enqueue_completion(completion, VDO_DEFAULT_Q_MAP_BIO_PRIORITY); +} + +static bool is_zero_block(char *block) +{ + int i; + + for (i = 0; i < VDO_BLOCK_SIZE; i += sizeof(u64)) { + if (*((u64 *) &block[i])) + return false; + } + + return true; +} + +static void copy_from_bio(struct bio *bio, char *data_ptr) +{ + struct bio_vec biovec; + struct bvec_iter iter; + + bio_for_each_segment(biovec, bio, iter) { + memcpy_from_bvec(data_ptr, &biovec); + data_ptr += biovec.bv_len; + } +} + +static void launch_bio(struct vdo *vdo, struct data_vio *data_vio, struct bio *bio) +{ + logical_block_number_t lbn; + /* + * Zero out the fields which don't need to be preserved (i.e. which are not pointers to + * separately allocated objects). + */ + memset(data_vio, 0, offsetof(struct data_vio, vio)); + memset(&data_vio->compression, 0, offsetof(struct compression_state, block)); + + data_vio->user_bio = bio; + data_vio->offset = to_bytes(bio->bi_iter.bi_sector & VDO_SECTORS_PER_BLOCK_MASK); + data_vio->is_partial = (bio->bi_iter.bi_size < VDO_BLOCK_SIZE) || (data_vio->offset != 0); + + /* + * Discards behave very differently than other requests when coming in from device-mapper. + * We have to be able to handle any size discards and various sector offsets within a + * block. + */ + if (bio_op(bio) == REQ_OP_DISCARD) { + data_vio->remaining_discard = bio->bi_iter.bi_size; + data_vio->write = true; + data_vio->is_discard = true; + if (data_vio->is_partial) { + vdo_count_bios(&vdo->stats.bios_in_partial, bio); + data_vio->read = true; + } + } else if (data_vio->is_partial) { + vdo_count_bios(&vdo->stats.bios_in_partial, bio); + data_vio->read = true; + if (bio_data_dir(bio) == WRITE) + data_vio->write = true; + } else if (bio_data_dir(bio) == READ) { + data_vio->read = true; + } else { + /* + * Copy the bio data to a char array so that we can continue to use the data after + * we acknowledge the bio. + */ + copy_from_bio(bio, data_vio->vio.data); + data_vio->is_zero = is_zero_block(data_vio->vio.data); + data_vio->write = true; + } + + if (data_vio->user_bio->bi_opf & REQ_FUA) + data_vio->fua = true; + + lbn = (bio->bi_iter.bi_sector - vdo->starting_sector_offset) / VDO_SECTORS_PER_BLOCK; + launch_data_vio(data_vio, lbn); +} + +static void assign_data_vio(struct limiter *limiter, struct data_vio *data_vio) +{ + struct bio *bio = bio_list_pop(limiter->permitted_waiters); + + launch_bio(limiter->pool->completion.vdo, data_vio, bio); + limiter->wake_count++; + + bio = bio_list_peek(limiter->permitted_waiters); + limiter->arrival = ((bio == NULL) ? U64_MAX : get_arrival_time(bio)); +} + +static void assign_discard_permit(struct limiter *limiter) +{ + struct bio *bio = bio_list_pop(&limiter->waiters); + + if (limiter->arrival == U64_MAX) + limiter->arrival = get_arrival_time(bio); + + bio_list_add(limiter->permitted_waiters, bio); +} + +static void get_waiters(struct limiter *limiter) +{ + bio_list_merge(&limiter->waiters, &limiter->new_waiters); + bio_list_init(&limiter->new_waiters); +} + +static inline struct data_vio *get_available_data_vio(struct data_vio_pool *pool) +{ + struct data_vio *data_vio = + list_first_entry(&pool->available, struct data_vio, pool_entry); + + list_del_init(&data_vio->pool_entry); + return data_vio; +} + +static void assign_data_vio_to_waiter(struct limiter *limiter) +{ + assign_data_vio(limiter, get_available_data_vio(limiter->pool)); +} + +static void update_limiter(struct limiter *limiter) +{ + struct bio_list *waiters = &limiter->waiters; + data_vio_count_t available = limiter->limit - limiter->busy; + + VDO_ASSERT_LOG_ONLY((limiter->release_count <= limiter->busy), + "Release count %u is not more than busy count %u", + limiter->release_count, limiter->busy); + + get_waiters(limiter); + for (; (limiter->release_count > 0) && !bio_list_empty(waiters); limiter->release_count--) + limiter->assigner(limiter); + + if (limiter->release_count > 0) { + WRITE_ONCE(limiter->busy, limiter->busy - limiter->release_count); + limiter->release_count = 0; + return; + } + + for (; (available > 0) && !bio_list_empty(waiters); available--) + limiter->assigner(limiter); + + WRITE_ONCE(limiter->busy, limiter->limit - available); + if (limiter->max_busy < limiter->busy) + WRITE_ONCE(limiter->max_busy, limiter->busy); +} + +/** + * schedule_releases() - Ensure that release processing is scheduled. + * + * If this call switches the state to processing, enqueue. Otherwise, some other thread has already + * done so. + */ +static void schedule_releases(struct data_vio_pool *pool) +{ + /* Pairs with the barrier in process_release_callback(). */ + smp_mb__before_atomic(); + if (atomic_cmpxchg(&pool->processing, false, true)) + return; + + pool->completion.requeue = true; + vdo_launch_completion_with_priority(&pool->completion, + CPU_Q_COMPLETE_VIO_PRIORITY); +} + +static void reuse_or_release_resources(struct data_vio_pool *pool, + struct data_vio *data_vio, + struct list_head *returned) +{ + if (data_vio->remaining_discard > 0) { + if (bio_list_empty(&pool->discard_limiter.waiters)) { + /* Return the data_vio's discard permit. */ + pool->discard_limiter.release_count++; + } else { + assign_discard_permit(&pool->discard_limiter); + } + } + + if (pool->limiter.arrival < pool->discard_limiter.arrival) { + assign_data_vio(&pool->limiter, data_vio); + } else if (pool->discard_limiter.arrival < U64_MAX) { + assign_data_vio(&pool->discard_limiter, data_vio); + } else { + list_add(&data_vio->pool_entry, returned); + pool->limiter.release_count++; + } +} + +/** + * process_release_callback() - Process a batch of data_vio releases. + * @completion: The pool with data_vios to release. + */ +static void process_release_callback(struct vdo_completion *completion) +{ + struct data_vio_pool *pool = as_data_vio_pool(completion); + bool reschedule; + bool drained; + data_vio_count_t processed; + data_vio_count_t to_wake; + data_vio_count_t discards_to_wake; + LIST_HEAD(returned); + + spin_lock(&pool->lock); + get_waiters(&pool->discard_limiter); + get_waiters(&pool->limiter); + spin_unlock(&pool->lock); + + if (pool->limiter.arrival == U64_MAX) { + struct bio *bio = bio_list_peek(&pool->limiter.waiters); + + if (bio != NULL) + pool->limiter.arrival = get_arrival_time(bio); + } + + for (processed = 0; processed < DATA_VIO_RELEASE_BATCH_SIZE; processed++) { + struct data_vio *data_vio; + struct funnel_queue_entry *entry = vdo_funnel_queue_poll(pool->queue); + + if (entry == NULL) + break; + + data_vio = as_data_vio(container_of(entry, struct vdo_completion, + work_queue_entry_link)); + acknowledge_data_vio(data_vio); + reuse_or_release_resources(pool, data_vio, &returned); + } + + spin_lock(&pool->lock); + /* + * There is a race where waiters could be added while we are in the unlocked section above. + * Those waiters could not see the resources we are now about to release, so we assign + * those resources now as we have no guarantee of being rescheduled. This is handled in + * update_limiter(). + */ + update_limiter(&pool->discard_limiter); + list_splice(&returned, &pool->available); + update_limiter(&pool->limiter); + to_wake = pool->limiter.wake_count; + pool->limiter.wake_count = 0; + discards_to_wake = pool->discard_limiter.wake_count; + pool->discard_limiter.wake_count = 0; + + atomic_set(&pool->processing, false); + /* Pairs with the barrier in schedule_releases(). */ + smp_mb(); + + reschedule = !vdo_is_funnel_queue_empty(pool->queue); + drained = (!reschedule && + vdo_is_state_draining(&pool->state) && + check_for_drain_complete_locked(pool)); + spin_unlock(&pool->lock); + + if (to_wake > 0) + wake_up_nr(&pool->limiter.blocked_threads, to_wake); + + if (discards_to_wake > 0) + wake_up_nr(&pool->discard_limiter.blocked_threads, discards_to_wake); + + if (reschedule) + schedule_releases(pool); + else if (drained) + vdo_finish_draining(&pool->state); +} + +static void initialize_limiter(struct limiter *limiter, struct data_vio_pool *pool, + assigner_fn assigner, data_vio_count_t limit) +{ + limiter->pool = pool; + limiter->assigner = assigner; + limiter->limit = limit; + limiter->arrival = U64_MAX; + init_waitqueue_head(&limiter->blocked_threads); +} + +/** + * initialize_data_vio() - Allocate the components of a data_vio. + * + * The caller is responsible for cleaning up the data_vio on error. + * + * Return: VDO_SUCCESS or an error. + */ +static int initialize_data_vio(struct data_vio *data_vio, struct vdo *vdo) +{ + struct bio *bio; + int result; + + BUILD_BUG_ON(VDO_BLOCK_SIZE > PAGE_SIZE); + result = vdo_allocate_memory(VDO_BLOCK_SIZE, 0, "data_vio data", + &data_vio->vio.data); + if (result != VDO_SUCCESS) + return vdo_log_error_strerror(result, + "data_vio data allocation failure"); + + result = vdo_allocate_memory(VDO_BLOCK_SIZE, 0, "compressed block", + &data_vio->compression.block); + if (result != VDO_SUCCESS) { + return vdo_log_error_strerror(result, + "data_vio compressed block allocation failure"); + } + + result = vdo_allocate_memory(VDO_BLOCK_SIZE, 0, "vio scratch", + &data_vio->scratch_block); + if (result != VDO_SUCCESS) + return vdo_log_error_strerror(result, + "data_vio scratch allocation failure"); + + result = vdo_create_bio(&bio); + if (result != VDO_SUCCESS) + return vdo_log_error_strerror(result, + "data_vio data bio allocation failure"); + + vdo_initialize_completion(&data_vio->decrement_completion, vdo, + VDO_DECREMENT_COMPLETION); + initialize_vio(&data_vio->vio, bio, 1, VIO_TYPE_DATA, VIO_PRIORITY_DATA, vdo); + + return VDO_SUCCESS; +} + +static void destroy_data_vio(struct data_vio *data_vio) +{ + if (data_vio == NULL) + return; + + vdo_free_bio(vdo_forget(data_vio->vio.bio)); + vdo_free(vdo_forget(data_vio->vio.data)); + vdo_free(vdo_forget(data_vio->compression.block)); + vdo_free(vdo_forget(data_vio->scratch_block)); +} + +/** + * make_data_vio_pool() - Initialize a data_vio pool. + * @vdo: The vdo to which the pool will belong. + * @pool_size: The number of data_vios in the pool. + * @discard_limit: The maximum number of data_vios which may be used for discards. + * @pool: A pointer to hold the newly allocated pool. + */ +int make_data_vio_pool(struct vdo *vdo, data_vio_count_t pool_size, + data_vio_count_t discard_limit, struct data_vio_pool **pool_ptr) +{ + int result; + struct data_vio_pool *pool; + data_vio_count_t i; + + result = vdo_allocate_extended(struct data_vio_pool, pool_size, struct data_vio, + __func__, &pool); + if (result != VDO_SUCCESS) + return result; + + VDO_ASSERT_LOG_ONLY((discard_limit <= pool_size), + "discard limit does not exceed pool size"); + initialize_limiter(&pool->discard_limiter, pool, assign_discard_permit, + discard_limit); + pool->discard_limiter.permitted_waiters = &pool->permitted_discards; + initialize_limiter(&pool->limiter, pool, assign_data_vio_to_waiter, pool_size); + pool->limiter.permitted_waiters = &pool->limiter.waiters; + INIT_LIST_HEAD(&pool->available); + spin_lock_init(&pool->lock); + vdo_set_admin_state_code(&pool->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + vdo_initialize_completion(&pool->completion, vdo, VDO_DATA_VIO_POOL_COMPLETION); + vdo_prepare_completion(&pool->completion, process_release_callback, + process_release_callback, vdo->thread_config.cpu_thread, + NULL); + + result = vdo_make_funnel_queue(&pool->queue); + if (result != VDO_SUCCESS) { + free_data_vio_pool(vdo_forget(pool)); + return result; + } + + for (i = 0; i < pool_size; i++) { + struct data_vio *data_vio = &pool->data_vios[i]; + + result = initialize_data_vio(data_vio, vdo); + if (result != VDO_SUCCESS) { + destroy_data_vio(data_vio); + free_data_vio_pool(pool); + return result; + } + + list_add(&data_vio->pool_entry, &pool->available); + } + + *pool_ptr = pool; + return VDO_SUCCESS; +} + +/** + * free_data_vio_pool() - Free a data_vio_pool and the data_vios in it. + * + * All data_vios must be returned to the pool before calling this function. + */ +void free_data_vio_pool(struct data_vio_pool *pool) +{ + struct data_vio *data_vio, *tmp; + + if (pool == NULL) + return; + + /* + * Pairs with the barrier in process_release_callback(). Possibly not needed since it + * caters to an enqueue vs. free race. + */ + smp_mb(); + BUG_ON(atomic_read(&pool->processing)); + + spin_lock(&pool->lock); + VDO_ASSERT_LOG_ONLY((pool->limiter.busy == 0), + "data_vio pool must not have %u busy entries when being freed", + pool->limiter.busy); + VDO_ASSERT_LOG_ONLY((bio_list_empty(&pool->limiter.waiters) && + bio_list_empty(&pool->limiter.new_waiters)), + "data_vio pool must not have threads waiting to read or write when being freed"); + VDO_ASSERT_LOG_ONLY((bio_list_empty(&pool->discard_limiter.waiters) && + bio_list_empty(&pool->discard_limiter.new_waiters)), + "data_vio pool must not have threads waiting to discard when being freed"); + spin_unlock(&pool->lock); + + list_for_each_entry_safe(data_vio, tmp, &pool->available, pool_entry) { + list_del_init(&data_vio->pool_entry); + destroy_data_vio(data_vio); + } + + vdo_free_funnel_queue(vdo_forget(pool->queue)); + vdo_free(pool); +} + +static bool acquire_permit(struct limiter *limiter) +{ + if (limiter->busy >= limiter->limit) + return false; + + WRITE_ONCE(limiter->busy, limiter->busy + 1); + if (limiter->max_busy < limiter->busy) + WRITE_ONCE(limiter->max_busy, limiter->busy); + return true; +} + +static void wait_permit(struct limiter *limiter, struct bio *bio) + __releases(&limiter->pool->lock) +{ + DEFINE_WAIT(wait); + + bio_list_add(&limiter->new_waiters, bio); + prepare_to_wait_exclusive(&limiter->blocked_threads, &wait, + TASK_UNINTERRUPTIBLE); + spin_unlock(&limiter->pool->lock); + io_schedule(); + finish_wait(&limiter->blocked_threads, &wait); +} + +/** + * vdo_launch_bio() - Acquire a data_vio from the pool, assign the bio to it, and launch it. + * + * This will block if data_vios or discard permits are not available. + */ +void vdo_launch_bio(struct data_vio_pool *pool, struct bio *bio) +{ + struct data_vio *data_vio; + + VDO_ASSERT_LOG_ONLY(!vdo_is_state_quiescent(&pool->state), + "data_vio_pool not quiescent on acquire"); + + bio->bi_private = (void *) jiffies; + spin_lock(&pool->lock); + if ((bio_op(bio) == REQ_OP_DISCARD) && + !acquire_permit(&pool->discard_limiter)) { + wait_permit(&pool->discard_limiter, bio); + return; + } + + if (!acquire_permit(&pool->limiter)) { + wait_permit(&pool->limiter, bio); + return; + } + + data_vio = get_available_data_vio(pool); + spin_unlock(&pool->lock); + launch_bio(pool->completion.vdo, data_vio, bio); +} + +/* Implements vdo_admin_initiator_fn. */ +static void initiate_drain(struct admin_state *state) +{ + bool drained; + struct data_vio_pool *pool = container_of(state, struct data_vio_pool, state); + + spin_lock(&pool->lock); + drained = check_for_drain_complete_locked(pool); + spin_unlock(&pool->lock); + + if (drained) + vdo_finish_draining(state); +} + +static void assert_on_vdo_cpu_thread(const struct vdo *vdo, const char *name) +{ + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == vdo->thread_config.cpu_thread), + "%s called on cpu thread", name); +} + +/** + * drain_data_vio_pool() - Wait asynchronously for all data_vios to be returned to the pool. + * @completion: The completion to notify when the pool has drained. + */ +void drain_data_vio_pool(struct data_vio_pool *pool, struct vdo_completion *completion) +{ + assert_on_vdo_cpu_thread(completion->vdo, __func__); + vdo_start_draining(&pool->state, VDO_ADMIN_STATE_SUSPENDING, completion, + initiate_drain); +} + +/** + * resume_data_vio_pool() - Resume a data_vio pool. + * @completion: The completion to notify when the pool has resumed. + */ +void resume_data_vio_pool(struct data_vio_pool *pool, struct vdo_completion *completion) +{ + assert_on_vdo_cpu_thread(completion->vdo, __func__); + vdo_continue_completion(completion, vdo_resume_if_quiescent(&pool->state)); +} + +static void dump_limiter(const char *name, struct limiter *limiter) +{ + vdo_log_info("%s: %u of %u busy (max %u), %s", name, limiter->busy, + limiter->limit, limiter->max_busy, + ((bio_list_empty(&limiter->waiters) && + bio_list_empty(&limiter->new_waiters)) ? + "no waiters" : "has waiters")); +} + +/** + * dump_data_vio_pool() - Dump a data_vio pool to the log. + * @dump_vios: Whether to dump the details of each busy data_vio as well. + */ +void dump_data_vio_pool(struct data_vio_pool *pool, bool dump_vios) +{ + /* + * In order that syslog can empty its buffer, sleep after 35 elements for 4ms (till the + * second clock tick). These numbers were picked based on experiments with lab machines. + */ + static const int ELEMENTS_PER_BATCH = 35; + static const int SLEEP_FOR_SYSLOG = 4000; + + if (pool == NULL) + return; + + spin_lock(&pool->lock); + dump_limiter("data_vios", &pool->limiter); + dump_limiter("discard permits", &pool->discard_limiter); + if (dump_vios) { + int i; + int dumped = 0; + + for (i = 0; i < pool->limiter.limit; i++) { + struct data_vio *data_vio = &pool->data_vios[i]; + + if (!list_empty(&data_vio->pool_entry)) + continue; + + dump_data_vio(data_vio); + if (++dumped >= ELEMENTS_PER_BATCH) { + spin_unlock(&pool->lock); + dumped = 0; + fsleep(SLEEP_FOR_SYSLOG); + spin_lock(&pool->lock); + } + } + } + + spin_unlock(&pool->lock); +} + +data_vio_count_t get_data_vio_pool_active_discards(struct data_vio_pool *pool) +{ + return READ_ONCE(pool->discard_limiter.busy); +} + +data_vio_count_t get_data_vio_pool_discard_limit(struct data_vio_pool *pool) +{ + return READ_ONCE(pool->discard_limiter.limit); +} + +data_vio_count_t get_data_vio_pool_maximum_discards(struct data_vio_pool *pool) +{ + return READ_ONCE(pool->discard_limiter.max_busy); +} + +int set_data_vio_pool_discard_limit(struct data_vio_pool *pool, data_vio_count_t limit) +{ + if (get_data_vio_pool_request_limit(pool) < limit) { + // The discard limit may not be higher than the data_vio limit. + return -EINVAL; + } + + spin_lock(&pool->lock); + pool->discard_limiter.limit = limit; + spin_unlock(&pool->lock); + + return VDO_SUCCESS; +} + +data_vio_count_t get_data_vio_pool_active_requests(struct data_vio_pool *pool) +{ + return READ_ONCE(pool->limiter.busy); +} + +data_vio_count_t get_data_vio_pool_request_limit(struct data_vio_pool *pool) +{ + return READ_ONCE(pool->limiter.limit); +} + +data_vio_count_t get_data_vio_pool_maximum_requests(struct data_vio_pool *pool) +{ + return READ_ONCE(pool->limiter.max_busy); +} + +static void update_data_vio_error_stats(struct data_vio *data_vio) +{ + u8 index = 0; + static const char * const operations[] = { + [0] = "empty", + [1] = "read", + [2] = "write", + [3] = "read-modify-write", + [5] = "read+fua", + [6] = "write+fua", + [7] = "read-modify-write+fua", + }; + + if (data_vio->read) + index = 1; + + if (data_vio->write) + index += 2; + + if (data_vio->fua) + index += 4; + + update_vio_error_stats(&data_vio->vio, + "Completing %s vio for LBN %llu with error after %s", + operations[index], + (unsigned long long) data_vio->logical.lbn, + get_data_vio_operation_name(data_vio)); +} + +static void perform_cleanup_stage(struct data_vio *data_vio, + enum data_vio_cleanup_stage stage); + +/** + * release_allocated_lock() - Release the PBN lock and/or the reference on the allocated block at + * the end of processing a data_vio. + */ +static void release_allocated_lock(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_allocated_zone(data_vio); + release_data_vio_allocation_lock(data_vio, false); + perform_cleanup_stage(data_vio, VIO_RELEASE_RECOVERY_LOCKS); +} + +/** release_lock() - Release an uncontended LBN lock. */ +static void release_lock(struct data_vio *data_vio, struct lbn_lock *lock) +{ + struct int_map *lock_map = lock->zone->lbn_operations; + struct data_vio *lock_holder; + + if (!lock->locked) { + /* The lock is not locked, so it had better not be registered in the lock map. */ + struct data_vio *lock_holder = vdo_int_map_get(lock_map, lock->lbn); + + VDO_ASSERT_LOG_ONLY((data_vio != lock_holder), + "no logical block lock held for block %llu", + (unsigned long long) lock->lbn); + return; + } + + /* Release the lock by removing the lock from the map. */ + lock_holder = vdo_int_map_remove(lock_map, lock->lbn); + VDO_ASSERT_LOG_ONLY((data_vio == lock_holder), + "logical block lock mismatch for block %llu", + (unsigned long long) lock->lbn); + lock->locked = false; +} + +/** transfer_lock() - Transfer a contended LBN lock to the eldest waiter. */ +static void transfer_lock(struct data_vio *data_vio, struct lbn_lock *lock) +{ + struct data_vio *lock_holder, *next_lock_holder; + int result; + + VDO_ASSERT_LOG_ONLY(lock->locked, "lbn_lock with waiters is not locked"); + + /* Another data_vio is waiting for the lock, transfer it in a single lock map operation. */ + next_lock_holder = + vdo_waiter_as_data_vio(vdo_waitq_dequeue_waiter(&lock->waiters)); + + /* Transfer the remaining lock waiters to the next lock holder. */ + vdo_waitq_transfer_all_waiters(&lock->waiters, + &next_lock_holder->logical.waiters); + + result = vdo_int_map_put(lock->zone->lbn_operations, lock->lbn, + next_lock_holder, true, (void **) &lock_holder); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(next_lock_holder, result); + return; + } + + VDO_ASSERT_LOG_ONLY((lock_holder == data_vio), + "logical block lock mismatch for block %llu", + (unsigned long long) lock->lbn); + lock->locked = false; + + /* + * If there are still waiters, other data_vios must be trying to get the lock we just + * transferred. We must ensure that the new lock holder doesn't block in the packer. + */ + if (vdo_waitq_has_waiters(&next_lock_holder->logical.waiters)) + cancel_data_vio_compression(next_lock_holder); + + /* + * Avoid stack overflow on lock transfer. + * FIXME: this is only an issue in the 1 thread config. + */ + next_lock_holder->vio.completion.requeue = true; + launch_locked_request(next_lock_holder); +} + +/** + * release_logical_lock() - Release the logical block lock and flush generation lock at the end of + * processing a data_vio. + */ +static void release_logical_lock(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + struct lbn_lock *lock = &data_vio->logical; + + assert_data_vio_in_logical_zone(data_vio); + + if (vdo_waitq_has_waiters(&lock->waiters)) + transfer_lock(data_vio, lock); + else + release_lock(data_vio, lock); + + vdo_release_flush_generation_lock(data_vio); + perform_cleanup_stage(data_vio, VIO_CLEANUP_DONE); +} + +/** clean_hash_lock() - Release the hash lock at the end of processing a data_vio. */ +static void clean_hash_lock(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_hash_zone(data_vio); + if (completion->result != VDO_SUCCESS) { + vdo_clean_failed_hash_lock(data_vio); + return; + } + + vdo_release_hash_lock(data_vio); + perform_cleanup_stage(data_vio, VIO_RELEASE_LOGICAL); +} + +/** + * finish_cleanup() - Make some assertions about a data_vio which has finished cleaning up. + * + * If it is part of a multi-block discard, starts on the next block, otherwise, returns it to the + * pool. + */ +static void finish_cleanup(struct data_vio *data_vio) +{ + struct vdo_completion *completion = &data_vio->vio.completion; + + VDO_ASSERT_LOG_ONLY(data_vio->allocation.lock == NULL, + "complete data_vio has no allocation lock"); + VDO_ASSERT_LOG_ONLY(data_vio->hash_lock == NULL, + "complete data_vio has no hash lock"); + if ((data_vio->remaining_discard <= VDO_BLOCK_SIZE) || + (completion->result != VDO_SUCCESS)) { + struct data_vio_pool *pool = completion->vdo->data_vio_pool; + + vdo_funnel_queue_put(pool->queue, &completion->work_queue_entry_link); + schedule_releases(pool); + return; + } + + data_vio->remaining_discard -= min_t(u32, data_vio->remaining_discard, + VDO_BLOCK_SIZE - data_vio->offset); + data_vio->is_partial = (data_vio->remaining_discard < VDO_BLOCK_SIZE); + data_vio->read = data_vio->is_partial; + data_vio->offset = 0; + completion->requeue = true; + launch_data_vio(data_vio, data_vio->logical.lbn + 1); +} + +/** perform_cleanup_stage() - Perform the next step in the process of cleaning up a data_vio. */ +static void perform_cleanup_stage(struct data_vio *data_vio, + enum data_vio_cleanup_stage stage) +{ + struct vdo *vdo = vdo_from_data_vio(data_vio); + + switch (stage) { + case VIO_RELEASE_HASH_LOCK: + if (data_vio->hash_lock != NULL) { + launch_data_vio_hash_zone_callback(data_vio, clean_hash_lock); + return; + } + fallthrough; + + case VIO_RELEASE_ALLOCATED: + if (data_vio_has_allocation(data_vio)) { + launch_data_vio_allocated_zone_callback(data_vio, + release_allocated_lock); + return; + } + fallthrough; + + case VIO_RELEASE_RECOVERY_LOCKS: + if ((data_vio->recovery_sequence_number > 0) && + (READ_ONCE(vdo->read_only_notifier.read_only_error) == VDO_SUCCESS) && + (data_vio->vio.completion.result != VDO_READ_ONLY)) + vdo_log_warning("VDO not read-only when cleaning data_vio with RJ lock"); + fallthrough; + + case VIO_RELEASE_LOGICAL: + launch_data_vio_logical_callback(data_vio, release_logical_lock); + return; + + default: + finish_cleanup(data_vio); + } +} + +void complete_data_vio(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + completion->error_handler = NULL; + data_vio->last_async_operation = VIO_ASYNC_OP_CLEANUP; + perform_cleanup_stage(data_vio, + (data_vio->write ? VIO_CLEANUP_START : VIO_RELEASE_LOGICAL)); +} + +static void enter_read_only_mode(struct vdo_completion *completion) +{ + if (vdo_is_read_only(completion->vdo)) + return; + + if (completion->result != VDO_READ_ONLY) { + struct data_vio *data_vio = as_data_vio(completion); + + vdo_log_error_strerror(completion->result, + "Preparing to enter read-only mode: data_vio for LBN %llu (becoming mapped to %llu, previously mapped to %llu, allocated %llu) is completing with a fatal error after operation %s", + (unsigned long long) data_vio->logical.lbn, + (unsigned long long) data_vio->new_mapped.pbn, + (unsigned long long) data_vio->mapped.pbn, + (unsigned long long) data_vio->allocation.pbn, + get_data_vio_operation_name(data_vio)); + } + + vdo_enter_read_only_mode(completion->vdo, completion->result); +} + +void handle_data_vio_error(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + if ((completion->result == VDO_READ_ONLY) || (data_vio->user_bio == NULL)) + enter_read_only_mode(completion); + + update_data_vio_error_stats(data_vio); + complete_data_vio(completion); +} + +/** + * get_data_vio_operation_name() - Get the name of the last asynchronous operation performed on a + * data_vio. + */ +const char *get_data_vio_operation_name(struct data_vio *data_vio) +{ + BUILD_BUG_ON((MAX_VIO_ASYNC_OPERATION_NUMBER - MIN_VIO_ASYNC_OPERATION_NUMBER) != + ARRAY_SIZE(ASYNC_OPERATION_NAMES)); + + return ((data_vio->last_async_operation < MAX_VIO_ASYNC_OPERATION_NUMBER) ? + ASYNC_OPERATION_NAMES[data_vio->last_async_operation] : + "unknown async operation"); +} + +/** + * data_vio_allocate_data_block() - Allocate a data block. + * + * @write_lock_type: The type of write lock to obtain on the block. + * @callback: The callback which will attempt an allocation in the current zone and continue if it + * succeeds. + * @error_handler: The handler for errors while allocating. + */ +void data_vio_allocate_data_block(struct data_vio *data_vio, + enum pbn_lock_type write_lock_type, + vdo_action_fn callback, vdo_action_fn error_handler) +{ + struct allocation *allocation = &data_vio->allocation; + + VDO_ASSERT_LOG_ONLY((allocation->pbn == VDO_ZERO_BLOCK), + "data_vio does not have an allocation"); + allocation->write_lock_type = write_lock_type; + allocation->zone = vdo_get_next_allocation_zone(data_vio->logical.zone); + allocation->first_allocation_zone = allocation->zone->zone_number; + + data_vio->vio.completion.error_handler = error_handler; + launch_data_vio_allocated_zone_callback(data_vio, callback); +} + +/** + * release_data_vio_allocation_lock() - Release the PBN lock on a data_vio's allocated block. + * @reset: If true, the allocation will be reset (i.e. any allocated pbn will be forgotten). + * + * If the reference to the locked block is still provisional, it will be released as well. + */ +void release_data_vio_allocation_lock(struct data_vio *data_vio, bool reset) +{ + struct allocation *allocation = &data_vio->allocation; + physical_block_number_t locked_pbn = allocation->pbn; + + assert_data_vio_in_allocated_zone(data_vio); + + if (reset || vdo_pbn_lock_has_provisional_reference(allocation->lock)) + allocation->pbn = VDO_ZERO_BLOCK; + + vdo_release_physical_zone_pbn_lock(allocation->zone, locked_pbn, + vdo_forget(allocation->lock)); +} + +/** + * uncompress_data_vio() - Uncompress the data a data_vio has just read. + * @mapping_state: The mapping state indicating which fragment to decompress. + * @buffer: The buffer to receive the uncompressed data. + */ +int uncompress_data_vio(struct data_vio *data_vio, + enum block_mapping_state mapping_state, char *buffer) +{ + int size; + u16 fragment_offset, fragment_size; + struct compressed_block *block = data_vio->compression.block; + int result = vdo_get_compressed_block_fragment(mapping_state, block, + &fragment_offset, &fragment_size); + + if (result != VDO_SUCCESS) { + vdo_log_debug("%s: compressed fragment error %d", __func__, result); + return result; + } + + size = LZ4_decompress_safe((block->data + fragment_offset), buffer, + fragment_size, VDO_BLOCK_SIZE); + if (size != VDO_BLOCK_SIZE) { + vdo_log_debug("%s: lz4 error", __func__); + return VDO_INVALID_FRAGMENT; + } + + return VDO_SUCCESS; +} + +/** + * modify_for_partial_write() - Do the modify-write part of a read-modify-write cycle. + * @completion: The data_vio which has just finished its read. + * + * This callback is registered in read_block(). + */ +static void modify_for_partial_write(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + char *data = data_vio->vio.data; + struct bio *bio = data_vio->user_bio; + + assert_data_vio_on_cpu_thread(data_vio); + + if (bio_op(bio) == REQ_OP_DISCARD) { + memset(data + data_vio->offset, '\0', min_t(u32, + data_vio->remaining_discard, + VDO_BLOCK_SIZE - data_vio->offset)); + } else { + copy_from_bio(bio, data + data_vio->offset); + } + + data_vio->is_zero = is_zero_block(data); + data_vio->read = false; + launch_data_vio_logical_callback(data_vio, + continue_data_vio_with_block_map_slot); +} + +static void complete_read(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + char *data = data_vio->vio.data; + bool compressed = vdo_is_state_compressed(data_vio->mapped.state); + + assert_data_vio_on_cpu_thread(data_vio); + + if (compressed) { + int result = uncompress_data_vio(data_vio, data_vio->mapped.state, data); + + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(data_vio, result); + return; + } + } + + if (data_vio->write) { + modify_for_partial_write(completion); + return; + } + + if (compressed || data_vio->is_partial) + copy_to_bio(data_vio->user_bio, data + data_vio->offset); + + acknowledge_data_vio(data_vio); + complete_data_vio(completion); +} + +static void read_endio(struct bio *bio) +{ + struct data_vio *data_vio = vio_as_data_vio(bio->bi_private); + int result = blk_status_to_errno(bio->bi_status); + + vdo_count_completed_bios(bio); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(data_vio, result); + return; + } + + launch_data_vio_cpu_callback(data_vio, complete_read, + CPU_Q_COMPLETE_READ_PRIORITY); +} + +static void complete_zero_read(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_on_cpu_thread(data_vio); + + if (data_vio->is_partial) { + memset(data_vio->vio.data, 0, VDO_BLOCK_SIZE); + if (data_vio->write) { + modify_for_partial_write(completion); + return; + } + } else { + zero_fill_bio(data_vio->user_bio); + } + + complete_read(completion); +} + +/** + * read_block() - Read a block asynchronously. + * + * This is the callback registered in read_block_mapping(). + */ +static void read_block(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + struct vio *vio = as_vio(completion); + int result = VDO_SUCCESS; + + if (data_vio->mapped.pbn == VDO_ZERO_BLOCK) { + launch_data_vio_cpu_callback(data_vio, complete_zero_read, + CPU_Q_COMPLETE_VIO_PRIORITY); + return; + } + + data_vio->last_async_operation = VIO_ASYNC_OP_READ_DATA_VIO; + if (vdo_is_state_compressed(data_vio->mapped.state)) { + result = vio_reset_bio(vio, (char *) data_vio->compression.block, + read_endio, REQ_OP_READ, data_vio->mapped.pbn); + } else { + blk_opf_t opf = ((data_vio->user_bio->bi_opf & PASSTHROUGH_FLAGS) | REQ_OP_READ); + + if (data_vio->is_partial) { + result = vio_reset_bio(vio, vio->data, read_endio, opf, + data_vio->mapped.pbn); + } else { + /* A full 4k read. Use the incoming bio to avoid having to copy the data */ + bio_reset(vio->bio, vio->bio->bi_bdev, opf); + bio_init_clone(data_vio->user_bio->bi_bdev, vio->bio, + data_vio->user_bio, GFP_KERNEL); + + /* Copy over the original bio iovec and opflags. */ + vdo_set_bio_properties(vio->bio, vio, read_endio, opf, + data_vio->mapped.pbn); + } + } + + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(data_vio, result); + return; + } + + vdo_submit_data_vio(data_vio); +} + +static inline struct data_vio * +reference_count_update_completion_as_data_vio(struct vdo_completion *completion) +{ + if (completion->type == VIO_COMPLETION) + return as_data_vio(completion); + + return container_of(completion, struct data_vio, decrement_completion); +} + +/** + * update_block_map() - Rendezvous of the data_vio and decrement completions after each has + * made its reference updates. Handle any error from either, or proceed + * to updating the block map. + * @completion: The completion of the write in progress. + */ +static void update_block_map(struct vdo_completion *completion) +{ + struct data_vio *data_vio = reference_count_update_completion_as_data_vio(completion); + + assert_data_vio_in_logical_zone(data_vio); + + if (!data_vio->first_reference_operation_complete) { + /* Rendezvous, we're first */ + data_vio->first_reference_operation_complete = true; + return; + } + + completion = &data_vio->vio.completion; + vdo_set_completion_result(completion, data_vio->decrement_completion.result); + if (completion->result != VDO_SUCCESS) { + handle_data_vio_error(completion); + return; + } + + completion->error_handler = handle_data_vio_error; + if (data_vio->hash_lock != NULL) + set_data_vio_hash_zone_callback(data_vio, vdo_continue_hash_lock); + else + completion->callback = complete_data_vio; + + data_vio->last_async_operation = VIO_ASYNC_OP_PUT_MAPPED_BLOCK; + vdo_put_mapped_block(data_vio); +} + +static void decrement_reference_count(struct vdo_completion *completion) +{ + struct data_vio *data_vio = container_of(completion, struct data_vio, + decrement_completion); + + assert_data_vio_in_mapped_zone(data_vio); + + vdo_set_completion_callback(completion, update_block_map, + data_vio->logical.zone->thread_id); + completion->error_handler = update_block_map; + vdo_modify_reference_count(completion, &data_vio->decrement_updater); +} + +static void increment_reference_count(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_new_mapped_zone(data_vio); + + if (data_vio->downgrade_allocation_lock) { + /* + * Now that the data has been written, it's safe to deduplicate against the + * block. Downgrade the allocation lock to a read lock so it can be used later by + * the hash lock. This is done here since it needs to happen sometime before we + * return to the hash zone, and we are currently on the correct thread. For + * compressed blocks, the downgrade will have already been done. + */ + vdo_downgrade_pbn_write_lock(data_vio->allocation.lock, false); + } + + set_data_vio_logical_callback(data_vio, update_block_map); + completion->error_handler = update_block_map; + vdo_modify_reference_count(completion, &data_vio->increment_updater); +} + +/** journal_remapping() - Add a recovery journal entry for a data remapping. */ +static void journal_remapping(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_journal_zone(data_vio); + + data_vio->decrement_updater.operation = VDO_JOURNAL_DATA_REMAPPING; + data_vio->decrement_updater.zpbn = data_vio->mapped; + if (data_vio->new_mapped.pbn == VDO_ZERO_BLOCK) { + data_vio->first_reference_operation_complete = true; + if (data_vio->mapped.pbn == VDO_ZERO_BLOCK) + set_data_vio_logical_callback(data_vio, update_block_map); + } else { + set_data_vio_new_mapped_zone_callback(data_vio, + increment_reference_count); + } + + if (data_vio->mapped.pbn == VDO_ZERO_BLOCK) { + data_vio->first_reference_operation_complete = true; + } else { + vdo_set_completion_callback(&data_vio->decrement_completion, + decrement_reference_count, + data_vio->mapped.zone->thread_id); + } + + data_vio->last_async_operation = VIO_ASYNC_OP_JOURNAL_REMAPPING; + vdo_add_recovery_journal_entry(completion->vdo->recovery_journal, data_vio); +} + +/** + * read_old_block_mapping() - Get the previous PBN/LBN mapping of an in-progress write. + * + * Gets the previous PBN mapped to this LBN from the block map, so as to make an appropriate + * journal entry referencing the removal of this LBN->PBN mapping. + */ +static void read_old_block_mapping(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_logical_zone(data_vio); + + data_vio->last_async_operation = VIO_ASYNC_OP_GET_MAPPED_BLOCK_FOR_WRITE; + set_data_vio_journal_callback(data_vio, journal_remapping); + vdo_get_mapped_block(data_vio); +} + +void update_metadata_for_data_vio_write(struct data_vio *data_vio, struct pbn_lock *lock) +{ + data_vio->increment_updater = (struct reference_updater) { + .operation = VDO_JOURNAL_DATA_REMAPPING, + .increment = true, + .zpbn = data_vio->new_mapped, + .lock = lock, + }; + + launch_data_vio_logical_callback(data_vio, read_old_block_mapping); +} + +/** + * pack_compressed_data() - Attempt to pack the compressed data_vio into a block. + * + * This is the callback registered in launch_compress_data_vio(). + */ +static void pack_compressed_data(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_packer_zone(data_vio); + + if (!vdo_get_compressing(vdo_from_data_vio(data_vio)) || + get_data_vio_compression_status(data_vio).may_not_compress) { + write_data_vio(data_vio); + return; + } + + data_vio->last_async_operation = VIO_ASYNC_OP_ATTEMPT_PACKING; + vdo_attempt_packing(data_vio); +} + +/** + * compress_data_vio() - Do the actual work of compressing the data on a CPU queue. + * + * This callback is registered in launch_compress_data_vio(). + */ +static void compress_data_vio(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + int size; + + assert_data_vio_on_cpu_thread(data_vio); + + /* + * By putting the compressed data at the start of the compressed block data field, we won't + * need to copy it if this data_vio becomes a compressed write agent. + */ + size = LZ4_compress_default(data_vio->vio.data, + data_vio->compression.block->data, VDO_BLOCK_SIZE, + VDO_MAX_COMPRESSED_FRAGMENT_SIZE, + (char *) vdo_get_work_queue_private_data()); + if ((size > 0) && (size < VDO_COMPRESSED_BLOCK_DATA_SIZE)) { + data_vio->compression.size = size; + launch_data_vio_packer_callback(data_vio, pack_compressed_data); + return; + } + + write_data_vio(data_vio); +} + +/** + * launch_compress_data_vio() - Continue a write by attempting to compress the data. + * + * This is a re-entry point to vio_write used by hash locks. + */ +void launch_compress_data_vio(struct data_vio *data_vio) +{ + VDO_ASSERT_LOG_ONLY(!data_vio->is_duplicate, "compressing a non-duplicate block"); + VDO_ASSERT_LOG_ONLY(data_vio->hash_lock != NULL, + "data_vio to compress has a hash_lock"); + VDO_ASSERT_LOG_ONLY(data_vio_has_allocation(data_vio), + "data_vio to compress has an allocation"); + + /* + * There are 4 reasons why a data_vio which has reached this point will not be eligible for + * compression: + * + * 1) Since data_vios can block indefinitely in the packer, it would be bad to do so if the + * write request also requests FUA. + * + * 2) A data_vio should not be compressed when compression is disabled for the vdo. + * + * 3) A data_vio could be doing a partial write on behalf of a larger discard which has not + * yet been acknowledged and hence blocking in the packer would be bad. + * + * 4) Some other data_vio may be waiting on this data_vio in which case blocking in the + * packer would also be bad. + */ + if (data_vio->fua || + !vdo_get_compressing(vdo_from_data_vio(data_vio)) || + ((data_vio->user_bio != NULL) && (bio_op(data_vio->user_bio) == REQ_OP_DISCARD)) || + (advance_data_vio_compression_stage(data_vio).stage != DATA_VIO_COMPRESSING)) { + write_data_vio(data_vio); + return; + } + + data_vio->last_async_operation = VIO_ASYNC_OP_COMPRESS_DATA_VIO; + launch_data_vio_cpu_callback(data_vio, compress_data_vio, + CPU_Q_COMPRESS_BLOCK_PRIORITY); +} + +/** + * hash_data_vio() - Hash the data in a data_vio and set the hash zone (which also flags the record + * name as set). + + * This callback is registered in prepare_for_dedupe(). + */ +static void hash_data_vio(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_on_cpu_thread(data_vio); + VDO_ASSERT_LOG_ONLY(!data_vio->is_zero, "zero blocks should not be hashed"); + + murmurhash3_128(data_vio->vio.data, VDO_BLOCK_SIZE, 0x62ea60be, + &data_vio->record_name); + + data_vio->hash_zone = vdo_select_hash_zone(vdo_from_data_vio(data_vio)->hash_zones, + &data_vio->record_name); + data_vio->last_async_operation = VIO_ASYNC_OP_ACQUIRE_VDO_HASH_LOCK; + launch_data_vio_hash_zone_callback(data_vio, vdo_acquire_hash_lock); +} + +/** prepare_for_dedupe() - Prepare for the dedupe path after attempting to get an allocation. */ +static void prepare_for_dedupe(struct data_vio *data_vio) +{ + /* We don't care what thread we are on. */ + VDO_ASSERT_LOG_ONLY(!data_vio->is_zero, "must not prepare to dedupe zero blocks"); + + /* + * Before we can dedupe, we need to know the record name, so the first + * step is to hash the block data. + */ + data_vio->last_async_operation = VIO_ASYNC_OP_HASH_DATA_VIO; + launch_data_vio_cpu_callback(data_vio, hash_data_vio, CPU_Q_HASH_BLOCK_PRIORITY); +} + +/** + * write_bio_finished() - This is the bio_end_io function registered in write_block() to be called + * when a data_vio's write to the underlying storage has completed. + */ +static void write_bio_finished(struct bio *bio) +{ + struct data_vio *data_vio = vio_as_data_vio((struct vio *) bio->bi_private); + + vdo_count_completed_bios(bio); + vdo_set_completion_result(&data_vio->vio.completion, + blk_status_to_errno(bio->bi_status)); + data_vio->downgrade_allocation_lock = true; + update_metadata_for_data_vio_write(data_vio, data_vio->allocation.lock); +} + +/** write_data_vio() - Write a data block to storage without compression. */ +void write_data_vio(struct data_vio *data_vio) +{ + struct data_vio_compression_status status, new_status; + int result; + + if (!data_vio_has_allocation(data_vio)) { + /* + * There was no space to write this block and we failed to deduplicate or compress + * it. + */ + continue_data_vio_with_error(data_vio, VDO_NO_SPACE); + return; + } + + new_status = (struct data_vio_compression_status) { + .stage = DATA_VIO_POST_PACKER, + .may_not_compress = true, + }; + + do { + status = get_data_vio_compression_status(data_vio); + } while ((status.stage != DATA_VIO_POST_PACKER) && + !set_data_vio_compression_status(data_vio, status, new_status)); + + /* Write the data from the data block buffer. */ + result = vio_reset_bio(&data_vio->vio, data_vio->vio.data, + write_bio_finished, REQ_OP_WRITE, + data_vio->allocation.pbn); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(data_vio, result); + return; + } + + data_vio->last_async_operation = VIO_ASYNC_OP_WRITE_DATA_VIO; + vdo_submit_data_vio(data_vio); +} + +/** + * acknowledge_write_callback() - Acknowledge a write to the requestor. + * + * This callback is registered in allocate_block() and continue_write_with_block_map_slot(). + */ +static void acknowledge_write_callback(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + struct vdo *vdo = completion->vdo; + + VDO_ASSERT_LOG_ONLY((!vdo_uses_bio_ack_queue(vdo) || + (vdo_get_callback_thread_id() == vdo->thread_config.bio_ack_thread)), + "%s() called on bio ack queue", __func__); + VDO_ASSERT_LOG_ONLY(data_vio_has_flush_generation_lock(data_vio), + "write VIO to be acknowledged has a flush generation lock"); + acknowledge_data_vio(data_vio); + if (data_vio->new_mapped.pbn == VDO_ZERO_BLOCK) { + /* This is a zero write or discard */ + update_metadata_for_data_vio_write(data_vio, NULL); + return; + } + + prepare_for_dedupe(data_vio); +} + +/** + * allocate_block() - Attempt to allocate a block in the current allocation zone. + * + * This callback is registered in continue_write_with_block_map_slot(). + */ +static void allocate_block(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_allocated_zone(data_vio); + + if (!vdo_allocate_block_in_zone(data_vio)) + return; + + completion->error_handler = handle_data_vio_error; + WRITE_ONCE(data_vio->allocation_succeeded, true); + data_vio->new_mapped = (struct zoned_pbn) { + .zone = data_vio->allocation.zone, + .pbn = data_vio->allocation.pbn, + .state = VDO_MAPPING_STATE_UNCOMPRESSED, + }; + + if (data_vio->fua) { + prepare_for_dedupe(data_vio); + return; + } + + data_vio->last_async_operation = VIO_ASYNC_OP_ACKNOWLEDGE_WRITE; + launch_data_vio_on_bio_ack_queue(data_vio, acknowledge_write_callback); +} + +/** + * handle_allocation_error() - Handle an error attempting to allocate a block. + * + * This error handler is registered in continue_write_with_block_map_slot(). + */ +static void handle_allocation_error(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + if (completion->result == VDO_NO_SPACE) { + /* We failed to get an allocation, but we can try to dedupe. */ + vdo_reset_completion(completion); + completion->error_handler = handle_data_vio_error; + prepare_for_dedupe(data_vio); + return; + } + + /* We got a "real" error, not just a failure to allocate, so fail the request. */ + handle_data_vio_error(completion); +} + +static int assert_is_discard(struct data_vio *data_vio) +{ + int result = VDO_ASSERT(data_vio->is_discard, + "data_vio with no block map page is a discard"); + + return ((result == VDO_SUCCESS) ? result : VDO_READ_ONLY); +} + +/** + * continue_data_vio_with_block_map_slot() - Read the data_vio's mapping from the block map. + * + * This callback is registered in launch_read_data_vio(). + */ +void continue_data_vio_with_block_map_slot(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + + assert_data_vio_in_logical_zone(data_vio); + if (data_vio->read) { + set_data_vio_logical_callback(data_vio, read_block); + data_vio->last_async_operation = VIO_ASYNC_OP_GET_MAPPED_BLOCK_FOR_READ; + vdo_get_mapped_block(data_vio); + return; + } + + vdo_acquire_flush_generation_lock(data_vio); + + if (data_vio->tree_lock.tree_slots[0].block_map_slot.pbn == VDO_ZERO_BLOCK) { + /* + * This is a discard for a block on a block map page which has not been allocated, so + * there's nothing more we need to do. + */ + completion->callback = complete_data_vio; + continue_data_vio_with_error(data_vio, assert_is_discard(data_vio)); + return; + } + + /* + * We need an allocation if this is neither a full-block discard nor a + * full-block zero write. + */ + if (!data_vio->is_zero && (!data_vio->is_discard || data_vio->is_partial)) { + data_vio_allocate_data_block(data_vio, VIO_WRITE_LOCK, allocate_block, + handle_allocation_error); + return; + } + + + /* + * We don't need to write any data, so skip allocation and just update the block map and + * reference counts (via the journal). + */ + data_vio->new_mapped.pbn = VDO_ZERO_BLOCK; + if (data_vio->is_zero) + data_vio->new_mapped.state = VDO_MAPPING_STATE_UNCOMPRESSED; + + if (data_vio->remaining_discard > VDO_BLOCK_SIZE) { + /* This is not the final block of a discard so we can't acknowledge it yet. */ + update_metadata_for_data_vio_write(data_vio, NULL); + return; + } + + data_vio->last_async_operation = VIO_ASYNC_OP_ACKNOWLEDGE_WRITE; + launch_data_vio_on_bio_ack_queue(data_vio, acknowledge_write_callback); +} diff --git a/drivers/md/dm-vdo/data-vio.h b/drivers/md/dm-vdo/data-vio.h new file mode 100644 index 000000000000..25926b6cd98b --- /dev/null +++ b/drivers/md/dm-vdo/data-vio.h @@ -0,0 +1,670 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef DATA_VIO_H +#define DATA_VIO_H + +#include +#include +#include + +#include "permassert.h" + +#include "indexer.h" + +#include "block-map.h" +#include "completion.h" +#include "constants.h" +#include "dedupe.h" +#include "encodings.h" +#include "logical-zone.h" +#include "physical-zone.h" +#include "types.h" +#include "vdo.h" +#include "vio.h" +#include "wait-queue.h" + +/* Codes for describing the last asynchronous operation performed on a vio. */ +enum async_operation_number { + MIN_VIO_ASYNC_OPERATION_NUMBER, + VIO_ASYNC_OP_LAUNCH = MIN_VIO_ASYNC_OPERATION_NUMBER, + VIO_ASYNC_OP_ACKNOWLEDGE_WRITE, + VIO_ASYNC_OP_ACQUIRE_VDO_HASH_LOCK, + VIO_ASYNC_OP_ATTEMPT_LOGICAL_BLOCK_LOCK, + VIO_ASYNC_OP_LOCK_DUPLICATE_PBN, + VIO_ASYNC_OP_CHECK_FOR_DUPLICATION, + VIO_ASYNC_OP_CLEANUP, + VIO_ASYNC_OP_COMPRESS_DATA_VIO, + VIO_ASYNC_OP_FIND_BLOCK_MAP_SLOT, + VIO_ASYNC_OP_GET_MAPPED_BLOCK_FOR_READ, + VIO_ASYNC_OP_GET_MAPPED_BLOCK_FOR_WRITE, + VIO_ASYNC_OP_HASH_DATA_VIO, + VIO_ASYNC_OP_JOURNAL_REMAPPING, + VIO_ASYNC_OP_ATTEMPT_PACKING, + VIO_ASYNC_OP_PUT_MAPPED_BLOCK, + VIO_ASYNC_OP_READ_DATA_VIO, + VIO_ASYNC_OP_UPDATE_DEDUPE_INDEX, + VIO_ASYNC_OP_UPDATE_REFERENCE_COUNTS, + VIO_ASYNC_OP_VERIFY_DUPLICATION, + VIO_ASYNC_OP_WRITE_DATA_VIO, + MAX_VIO_ASYNC_OPERATION_NUMBER, +} __packed; + +struct lbn_lock { + logical_block_number_t lbn; + bool locked; + struct vdo_wait_queue waiters; + struct logical_zone *zone; +}; + +/* A position in the arboreal block map at a specific level. */ +struct block_map_tree_slot { + page_number_t page_index; + struct block_map_slot block_map_slot; +}; + +/* Fields for using the arboreal block map. */ +struct tree_lock { + /* The current height at which this data_vio is operating */ + height_t height; + /* The block map tree for this LBN */ + root_count_t root_index; + /* Whether we hold a page lock */ + bool locked; + /* The key for the lock map */ + u64 key; + /* The queue of waiters for the page this vio is allocating or loading */ + struct vdo_wait_queue waiters; + /* The block map tree slots for this LBN */ + struct block_map_tree_slot tree_slots[VDO_BLOCK_MAP_TREE_HEIGHT + 1]; +}; + +struct zoned_pbn { + physical_block_number_t pbn; + enum block_mapping_state state; + struct physical_zone *zone; +}; + +/* + * Where a data_vio is on the compression path; advance_compression_stage() depends on the order of + * this enum. + */ +enum data_vio_compression_stage { + /* A data_vio which has not yet entered the compression path */ + DATA_VIO_PRE_COMPRESSOR, + /* A data_vio which is in the compressor */ + DATA_VIO_COMPRESSING, + /* A data_vio which is blocked in the packer */ + DATA_VIO_PACKING, + /* A data_vio which is no longer on the compression path (and never will be) */ + DATA_VIO_POST_PACKER, +}; + +struct data_vio_compression_status { + enum data_vio_compression_stage stage; + bool may_not_compress; +}; + +struct compression_state { + /* + * The current compression status of this data_vio. This field contains a value which + * consists of a data_vio_compression_stage and a flag indicating whether a request has + * been made to cancel (or prevent) compression for this data_vio. + * + * This field should be accessed through the get_data_vio_compression_status() and + * set_data_vio_compression_status() methods. It should not be accessed directly. + */ + atomic_t status; + + /* The compressed size of this block */ + u16 size; + + /* The packer input or output bin slot which holds the enclosing data_vio */ + slot_number_t slot; + + /* The packer bin to which the enclosing data_vio has been assigned */ + struct packer_bin *bin; + + /* A link in the chain of data_vios which have been packed together */ + struct data_vio *next_in_batch; + + /* A vio which is blocked in the packer while holding a lock this vio needs. */ + struct data_vio *lock_holder; + + /* + * The compressed block used to hold the compressed form of this block and that of any + * other blocks for which this data_vio is the compressed write agent. + */ + struct compressed_block *block; +}; + +/* Fields supporting allocation of data blocks. */ +struct allocation { + /* The physical zone in which to allocate a physical block */ + struct physical_zone *zone; + + /* The block allocated to this vio */ + physical_block_number_t pbn; + + /* + * If non-NULL, the pooled PBN lock held on the allocated block. Must be a write lock until + * the block has been written, after which it will become a read lock. + */ + struct pbn_lock *lock; + + /* The type of write lock to obtain on the allocated block */ + enum pbn_lock_type write_lock_type; + + /* The zone which was the start of the current allocation cycle */ + zone_count_t first_allocation_zone; + + /* Whether this vio should wait for a clean slab */ + bool wait_for_clean_slab; +}; + +struct reference_updater { + enum journal_operation operation; + bool increment; + struct zoned_pbn zpbn; + struct pbn_lock *lock; + struct vdo_waiter waiter; +}; + +/* A vio for processing user data requests. */ +struct data_vio { + /* The vdo_wait_queue entry structure */ + struct vdo_waiter waiter; + + /* The logical block of this request */ + struct lbn_lock logical; + + /* The state for traversing the block map tree */ + struct tree_lock tree_lock; + + /* The current partition address of this block */ + struct zoned_pbn mapped; + + /* The hash of this vio (if not zero) */ + struct uds_record_name record_name; + + /* Used for logging and debugging */ + enum async_operation_number last_async_operation; + + /* The operations to record in the recovery and slab journals */ + struct reference_updater increment_updater; + struct reference_updater decrement_updater; + + u16 read : 1; + u16 write : 1; + u16 fua : 1; + u16 is_zero : 1; + u16 is_discard : 1; + u16 is_partial : 1; + u16 is_duplicate : 1; + u16 first_reference_operation_complete : 1; + u16 downgrade_allocation_lock : 1; + + struct allocation allocation; + + /* + * Whether this vio has received an allocation. This field is examined from threads not in + * the allocation zone. + */ + bool allocation_succeeded; + + /* The new partition address of this block after the vio write completes */ + struct zoned_pbn new_mapped; + + /* The hash zone responsible for the name (NULL if is_zero_block) */ + struct hash_zone *hash_zone; + + /* The lock this vio holds or shares with other vios with the same data */ + struct hash_lock *hash_lock; + + /* All data_vios sharing a hash lock are kept in a list linking these list entries */ + struct list_head hash_lock_entry; + + /* The block number in the partition of the UDS deduplication advice */ + struct zoned_pbn duplicate; + + /* + * The sequence number of the recovery journal block containing the increment entry for + * this vio. + */ + sequence_number_t recovery_sequence_number; + + /* The point in the recovery journal where this write last made an entry */ + struct journal_point recovery_journal_point; + + /* The list of vios in user initiated write requests */ + struct list_head write_entry; + + /* The generation number of the VDO that this vio belongs to */ + sequence_number_t flush_generation; + + /* The completion to use for fetching block map pages for this vio */ + struct vdo_page_completion page_completion; + + /* The user bio that initiated this VIO */ + struct bio *user_bio; + + /* partial block support */ + block_size_t offset; + + /* + * The number of bytes to be discarded. For discards, this field will always be positive, + * whereas for non-discards it will always be 0. Hence it can be used to determine whether + * a data_vio is processing a discard, even after the user_bio has been acknowledged. + */ + u32 remaining_discard; + + struct dedupe_context *dedupe_context; + + /* Fields beyond this point will not be reset when a pooled data_vio is reused. */ + + struct vio vio; + + /* The completion for making reference count decrements */ + struct vdo_completion decrement_completion; + + /* All of the fields necessary for the compression path */ + struct compression_state compression; + + /* A block used as output during compression or uncompression */ + char *scratch_block; + + struct list_head pool_entry; +}; + +static inline struct data_vio *vio_as_data_vio(struct vio *vio) +{ + VDO_ASSERT_LOG_ONLY((vio->type == VIO_TYPE_DATA), "vio is a data_vio"); + return container_of(vio, struct data_vio, vio); +} + +static inline struct data_vio *as_data_vio(struct vdo_completion *completion) +{ + return vio_as_data_vio(as_vio(completion)); +} + +static inline struct data_vio *vdo_waiter_as_data_vio(struct vdo_waiter *waiter) +{ + if (waiter == NULL) + return NULL; + + return container_of(waiter, struct data_vio, waiter); +} + +static inline struct data_vio *data_vio_from_reference_updater(struct reference_updater *updater) +{ + if (updater->increment) + return container_of(updater, struct data_vio, increment_updater); + + return container_of(updater, struct data_vio, decrement_updater); +} + +static inline bool data_vio_has_flush_generation_lock(struct data_vio *data_vio) +{ + return !list_empty(&data_vio->write_entry); +} + +static inline struct vdo *vdo_from_data_vio(struct data_vio *data_vio) +{ + return data_vio->vio.completion.vdo; +} + +static inline bool data_vio_has_allocation(struct data_vio *data_vio) +{ + return (data_vio->allocation.pbn != VDO_ZERO_BLOCK); +} + +struct data_vio_compression_status __must_check +advance_data_vio_compression_stage(struct data_vio *data_vio); +struct data_vio_compression_status __must_check +get_data_vio_compression_status(struct data_vio *data_vio); +bool cancel_data_vio_compression(struct data_vio *data_vio); + +struct data_vio_pool; + +int make_data_vio_pool(struct vdo *vdo, data_vio_count_t pool_size, + data_vio_count_t discard_limit, struct data_vio_pool **pool_ptr); +void free_data_vio_pool(struct data_vio_pool *pool); +void vdo_launch_bio(struct data_vio_pool *pool, struct bio *bio); +void drain_data_vio_pool(struct data_vio_pool *pool, struct vdo_completion *completion); +void resume_data_vio_pool(struct data_vio_pool *pool, struct vdo_completion *completion); + +void dump_data_vio_pool(struct data_vio_pool *pool, bool dump_vios); +data_vio_count_t get_data_vio_pool_active_discards(struct data_vio_pool *pool); +data_vio_count_t get_data_vio_pool_discard_limit(struct data_vio_pool *pool); +data_vio_count_t get_data_vio_pool_maximum_discards(struct data_vio_pool *pool); +int __must_check set_data_vio_pool_discard_limit(struct data_vio_pool *pool, + data_vio_count_t limit); +data_vio_count_t get_data_vio_pool_active_requests(struct data_vio_pool *pool); +data_vio_count_t get_data_vio_pool_request_limit(struct data_vio_pool *pool); +data_vio_count_t get_data_vio_pool_maximum_requests(struct data_vio_pool *pool); + +void complete_data_vio(struct vdo_completion *completion); +void handle_data_vio_error(struct vdo_completion *completion); + +static inline void continue_data_vio(struct data_vio *data_vio) +{ + vdo_launch_completion(&data_vio->vio.completion); +} + +/** + * continue_data_vio_with_error() - Set an error code and then continue processing a data_vio. + * + * This will not mask older errors. This function can be called with a success code, but it is more + * efficient to call continue_data_vio() if the caller knows the result was a success. + */ +static inline void continue_data_vio_with_error(struct data_vio *data_vio, int result) +{ + vdo_continue_completion(&data_vio->vio.completion, result); +} + +const char * __must_check get_data_vio_operation_name(struct data_vio *data_vio); + +static inline void assert_data_vio_in_hash_zone(struct data_vio *data_vio) +{ + thread_id_t expected = data_vio->hash_zone->thread_id; + thread_id_t thread_id = vdo_get_callback_thread_id(); + /* + * It's odd to use the LBN, but converting the record name to hex is a bit clunky for an + * inline, and the LBN better than nothing as an identifier. + */ + VDO_ASSERT_LOG_ONLY((expected == thread_id), + "data_vio for logical block %llu on thread %u, should be on hash zone thread %u", + (unsigned long long) data_vio->logical.lbn, thread_id, expected); +} + +static inline void set_data_vio_hash_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + vdo_set_completion_callback(&data_vio->vio.completion, callback, + data_vio->hash_zone->thread_id); +} + +/** + * launch_data_vio_hash_zone_callback() - Set a callback as a hash zone operation and invoke it + * immediately. + */ +static inline void launch_data_vio_hash_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + set_data_vio_hash_zone_callback(data_vio, callback); + vdo_launch_completion(&data_vio->vio.completion); +} + +static inline void assert_data_vio_in_logical_zone(struct data_vio *data_vio) +{ + thread_id_t expected = data_vio->logical.zone->thread_id; + thread_id_t thread_id = vdo_get_callback_thread_id(); + + VDO_ASSERT_LOG_ONLY((expected == thread_id), + "data_vio for logical block %llu on thread %u, should be on thread %u", + (unsigned long long) data_vio->logical.lbn, thread_id, expected); +} + +static inline void set_data_vio_logical_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + vdo_set_completion_callback(&data_vio->vio.completion, callback, + data_vio->logical.zone->thread_id); +} + +/** + * launch_data_vio_logical_callback() - Set a callback as a logical block operation and invoke it + * immediately. + */ +static inline void launch_data_vio_logical_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + set_data_vio_logical_callback(data_vio, callback); + vdo_launch_completion(&data_vio->vio.completion); +} + +static inline void assert_data_vio_in_allocated_zone(struct data_vio *data_vio) +{ + thread_id_t expected = data_vio->allocation.zone->thread_id; + thread_id_t thread_id = vdo_get_callback_thread_id(); + + VDO_ASSERT_LOG_ONLY((expected == thread_id), + "struct data_vio for allocated physical block %llu on thread %u, should be on thread %u", + (unsigned long long) data_vio->allocation.pbn, thread_id, + expected); +} + +static inline void set_data_vio_allocated_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + vdo_set_completion_callback(&data_vio->vio.completion, callback, + data_vio->allocation.zone->thread_id); +} + +/** + * launch_data_vio_allocated_zone_callback() - Set a callback as a physical block operation in a + * data_vio's allocated zone and queue the data_vio and + * invoke it immediately. + */ +static inline void launch_data_vio_allocated_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + set_data_vio_allocated_zone_callback(data_vio, callback); + vdo_launch_completion(&data_vio->vio.completion); +} + +static inline void assert_data_vio_in_duplicate_zone(struct data_vio *data_vio) +{ + thread_id_t expected = data_vio->duplicate.zone->thread_id; + thread_id_t thread_id = vdo_get_callback_thread_id(); + + VDO_ASSERT_LOG_ONLY((expected == thread_id), + "data_vio for duplicate physical block %llu on thread %u, should be on thread %u", + (unsigned long long) data_vio->duplicate.pbn, thread_id, + expected); +} + +static inline void set_data_vio_duplicate_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + vdo_set_completion_callback(&data_vio->vio.completion, callback, + data_vio->duplicate.zone->thread_id); +} + +/** + * launch_data_vio_duplicate_zone_callback() - Set a callback as a physical block operation in a + * data_vio's duplicate zone and queue the data_vio and + * invoke it immediately. + */ +static inline void launch_data_vio_duplicate_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + set_data_vio_duplicate_zone_callback(data_vio, callback); + vdo_launch_completion(&data_vio->vio.completion); +} + +static inline void assert_data_vio_in_mapped_zone(struct data_vio *data_vio) +{ + thread_id_t expected = data_vio->mapped.zone->thread_id; + thread_id_t thread_id = vdo_get_callback_thread_id(); + + VDO_ASSERT_LOG_ONLY((expected == thread_id), + "data_vio for mapped physical block %llu on thread %u, should be on thread %u", + (unsigned long long) data_vio->mapped.pbn, thread_id, expected); +} + +static inline void set_data_vio_mapped_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + vdo_set_completion_callback(&data_vio->vio.completion, callback, + data_vio->mapped.zone->thread_id); +} + +static inline void assert_data_vio_in_new_mapped_zone(struct data_vio *data_vio) +{ + thread_id_t expected = data_vio->new_mapped.zone->thread_id; + thread_id_t thread_id = vdo_get_callback_thread_id(); + + VDO_ASSERT_LOG_ONLY((expected == thread_id), + "data_vio for new_mapped physical block %llu on thread %u, should be on thread %u", + (unsigned long long) data_vio->new_mapped.pbn, thread_id, + expected); +} + +static inline void set_data_vio_new_mapped_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + vdo_set_completion_callback(&data_vio->vio.completion, callback, + data_vio->new_mapped.zone->thread_id); +} + +static inline void assert_data_vio_in_journal_zone(struct data_vio *data_vio) +{ + thread_id_t journal_thread = vdo_from_data_vio(data_vio)->thread_config.journal_thread; + thread_id_t thread_id = vdo_get_callback_thread_id(); + + VDO_ASSERT_LOG_ONLY((journal_thread == thread_id), + "data_vio for logical block %llu on thread %u, should be on journal thread %u", + (unsigned long long) data_vio->logical.lbn, thread_id, + journal_thread); +} + +static inline void set_data_vio_journal_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + thread_id_t journal_thread = vdo_from_data_vio(data_vio)->thread_config.journal_thread; + + vdo_set_completion_callback(&data_vio->vio.completion, callback, journal_thread); +} + +/** + * launch_data_vio_journal_callback() - Set a callback as a journal operation and invoke it + * immediately. + */ +static inline void launch_data_vio_journal_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + set_data_vio_journal_callback(data_vio, callback); + vdo_launch_completion(&data_vio->vio.completion); +} + +static inline void assert_data_vio_in_packer_zone(struct data_vio *data_vio) +{ + thread_id_t packer_thread = vdo_from_data_vio(data_vio)->thread_config.packer_thread; + thread_id_t thread_id = vdo_get_callback_thread_id(); + + VDO_ASSERT_LOG_ONLY((packer_thread == thread_id), + "data_vio for logical block %llu on thread %u, should be on packer thread %u", + (unsigned long long) data_vio->logical.lbn, thread_id, + packer_thread); +} + +static inline void set_data_vio_packer_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + thread_id_t packer_thread = vdo_from_data_vio(data_vio)->thread_config.packer_thread; + + vdo_set_completion_callback(&data_vio->vio.completion, callback, packer_thread); +} + +/** + * launch_data_vio_packer_callback() - Set a callback as a packer operation and invoke it + * immediately. + */ +static inline void launch_data_vio_packer_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + set_data_vio_packer_callback(data_vio, callback); + vdo_launch_completion(&data_vio->vio.completion); +} + +static inline void assert_data_vio_on_cpu_thread(struct data_vio *data_vio) +{ + thread_id_t cpu_thread = vdo_from_data_vio(data_vio)->thread_config.cpu_thread; + thread_id_t thread_id = vdo_get_callback_thread_id(); + + VDO_ASSERT_LOG_ONLY((cpu_thread == thread_id), + "data_vio for logical block %llu on thread %u, should be on cpu thread %u", + (unsigned long long) data_vio->logical.lbn, thread_id, + cpu_thread); +} + +static inline void set_data_vio_cpu_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + thread_id_t cpu_thread = vdo_from_data_vio(data_vio)->thread_config.cpu_thread; + + vdo_set_completion_callback(&data_vio->vio.completion, callback, cpu_thread); +} + +/** + * launch_data_vio_cpu_callback() - Set a callback to run on the CPU queues and invoke it + * immediately. + */ +static inline void launch_data_vio_cpu_callback(struct data_vio *data_vio, + vdo_action_fn callback, + enum vdo_completion_priority priority) +{ + set_data_vio_cpu_callback(data_vio, callback); + vdo_launch_completion_with_priority(&data_vio->vio.completion, priority); +} + +static inline void set_data_vio_bio_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + vdo_set_completion_callback(&data_vio->vio.completion, callback, + get_vio_bio_zone_thread_id(&data_vio->vio)); +} + +/** + * launch_data_vio_bio_zone_callback() - Set a callback as a bio zone operation and invoke it + * immediately. + */ +static inline void launch_data_vio_bio_zone_callback(struct data_vio *data_vio, + vdo_action_fn callback) +{ + set_data_vio_bio_zone_callback(data_vio, callback); + vdo_launch_completion_with_priority(&data_vio->vio.completion, + BIO_Q_DATA_PRIORITY); +} + +/** + * launch_data_vio_on_bio_ack_queue() - If the vdo uses a bio_ack queue, set a callback to run on + * it and invoke it immediately, otherwise, just run the + * callback on the current thread. + */ +static inline void launch_data_vio_on_bio_ack_queue(struct data_vio *data_vio, + vdo_action_fn callback) +{ + struct vdo_completion *completion = &data_vio->vio.completion; + struct vdo *vdo = completion->vdo; + + if (!vdo_uses_bio_ack_queue(vdo)) { + callback(completion); + return; + } + + vdo_set_completion_callback(completion, callback, + vdo->thread_config.bio_ack_thread); + vdo_launch_completion_with_priority(completion, BIO_ACK_Q_ACK_PRIORITY); +} + +void data_vio_allocate_data_block(struct data_vio *data_vio, + enum pbn_lock_type write_lock_type, + vdo_action_fn callback, vdo_action_fn error_handler); + +void release_data_vio_allocation_lock(struct data_vio *data_vio, bool reset); + +int __must_check uncompress_data_vio(struct data_vio *data_vio, + enum block_mapping_state mapping_state, + char *buffer); + +void update_metadata_for_data_vio_write(struct data_vio *data_vio, + struct pbn_lock *lock); +void write_data_vio(struct data_vio *data_vio); +void launch_compress_data_vio(struct data_vio *data_vio); +void continue_data_vio_with_block_map_slot(struct vdo_completion *completion); + +#endif /* DATA_VIO_H */ diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c new file mode 100644 index 000000000000..117266e1b3ae --- /dev/null +++ b/drivers/md/dm-vdo/dedupe.c @@ -0,0 +1,3003 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +/** + * DOC: + * + * Hash Locks: + * + * A hash_lock controls and coordinates writing, index access, and dedupe among groups of data_vios + * concurrently writing identical blocks, allowing them to deduplicate not only against advice but + * also against each other. This saves on index queries and allows those data_vios to concurrently + * deduplicate against a single block instead of being serialized through a PBN read lock. Only one + * index query is needed for each hash_lock, instead of one for every data_vio. + * + * Hash_locks are assigned to hash_zones by computing a modulus on the hash itself. Each hash_zone + * has a single dedicated queue and thread for performing all operations on the hash_locks assigned + * to that zone. The concurrency guarantees of this single-threaded model allow the code to omit + * more fine-grained locking for the hash_lock structures. + * + * A hash_lock acts like a state machine perhaps more than as a lock. Other than the starting and + * ending states INITIALIZING and BYPASSING, every state represents and is held for the duration of + * an asynchronous operation. All state transitions are performed on the thread of the hash_zone + * containing the lock. An asynchronous operation is almost always performed upon entering a state, + * and the callback from that operation triggers exiting the state and entering a new state. + * + * In all states except DEDUPING, there is a single data_vio, called the lock agent, performing the + * asynchronous operations on behalf of the lock. The agent will change during the lifetime of the + * lock if the lock is shared by more than one data_vio. data_vios waiting to deduplicate are kept + * on a wait queue. Viewed a different way, the agent holds the lock exclusively until the lock + * enters the DEDUPING state, at which point it becomes a shared lock that all the waiters (and any + * new data_vios that arrive) use to share a PBN lock. In state DEDUPING, there is no agent. When + * the last data_vio in the lock calls back in DEDUPING, it becomes the agent and the lock becomes + * exclusive again. New data_vios that arrive in the lock will also go on the wait queue. + * + * The existence of lock waiters is a key factor controlling which state the lock transitions to + * next. When the lock is new or has waiters, it will always try to reach DEDUPING, and when it + * doesn't, it will try to clean up and exit. + * + * Deduping requires holding a PBN lock on a block that is known to contain data identical to the + * data_vios in the lock, so the lock will send the agent to the duplicate zone to acquire the PBN + * lock (LOCKING), to the kernel I/O threads to read and verify the data (VERIFYING), or to write a + * new copy of the data to a full data block or a slot in a compressed block (WRITING). + * + * Cleaning up consists of updating the index when the data location is different from the initial + * index query (UPDATING, triggered by stale advice, compression, and rollover), releasing the PBN + * lock on the duplicate block (UNLOCKING), and if the agent is the last data_vio referencing the + * lock, releasing the hash_lock itself back to the hash zone (BYPASSING). + * + * The shortest sequence of states is for non-concurrent writes of new data: + * INITIALIZING -> QUERYING -> WRITING -> BYPASSING + * This sequence is short because no PBN read lock or index update is needed. + * + * Non-concurrent, finding valid advice looks like this (endpoints elided): + * -> QUERYING -> LOCKING -> VERIFYING -> DEDUPING -> UNLOCKING -> + * Or with stale advice (endpoints elided): + * -> QUERYING -> LOCKING -> VERIFYING -> UNLOCKING -> WRITING -> UPDATING -> + * + * When there are not enough available reference count increments available on a PBN for a data_vio + * to deduplicate, a new lock is forked and the excess waiters roll over to the new lock (which + * goes directly to WRITING). The new lock takes the place of the old lock in the lock map so new + * data_vios will be directed to it. The two locks will proceed independently, but only the new + * lock will have the right to update the index (unless it also forks). + * + * Since rollover happens in a lock instance, once a valid data location has been selected, it will + * not change. QUERYING and WRITING are only performed once per lock lifetime. All other + * non-endpoint states can be re-entered. + * + * The function names in this module follow a convention referencing the states and transitions in + * the state machine. For example, for the LOCKING state, there are start_locking() and + * finish_locking() functions. start_locking() is invoked by the finish function of the state (or + * states) that transition to LOCKING. It performs the actual lock state change and must be invoked + * on the hash zone thread. finish_locking() is called by (or continued via callback from) the + * code actually obtaining the lock. It does any bookkeeping or decision-making required and + * invokes the appropriate start function of the state being transitioned to after LOCKING. + * + * ---------------------------------------------------------------------- + * + * Index Queries: + * + * A query to the UDS index is handled asynchronously by the index's threads. When the query is + * complete, a callback supplied with the query will be called from one of the those threads. Under + * heavy system load, the index may be slower to respond than is desirable for reasonable I/O + * throughput. Since deduplication of writes is not necessary for correct operation of a VDO + * device, it is acceptable to timeout out slow index queries and proceed to fulfill a write + * request without deduplicating. However, because the uds_request struct itself is supplied by the + * caller, we can not simply reuse a uds_request object which we have chosen to timeout. Hence, + * each hash_zone maintains a pool of dedupe_contexts which each contain a uds_request along with a + * reference to the data_vio on behalf of which they are performing a query. + * + * When a hash_lock needs to query the index, it attempts to acquire an unused dedupe_context from + * its hash_zone's pool. If one is available, that context is prepared, associated with the + * hash_lock's agent, added to the list of pending contexts, and then sent to the index. The + * context's state will be transitioned from DEDUPE_CONTEXT_IDLE to DEDUPE_CONTEXT_PENDING. If all + * goes well, the dedupe callback will be called by the index which will change the context's state + * to DEDUPE_CONTEXT_COMPLETE, and the associated data_vio will be enqueued to run back in the hash + * zone where the query results will be processed and the context will be put back in the idle + * state and returned to the hash_zone's available list. + * + * The first time an index query is launched from a given hash_zone, a timer is started. When the + * timer fires, the hash_zone's completion is enqueued to run in the hash_zone where the zone's + * pending list will be searched for any contexts in the pending state which have been running for + * too long. Those contexts are transitioned to the DEDUPE_CONTEXT_TIMED_OUT state and moved to the + * zone's timed_out list where they won't be examined again if there is a subsequent time out). The + * data_vios associated with timed out contexts are sent to continue processing their write + * operation without deduplicating. The timer is also restarted. + * + * When the dedupe callback is run for a context which is in the timed out state, that context is + * moved to the DEDUPE_CONTEXT_TIMED_OUT_COMPLETE state. No other action need be taken as the + * associated data_vios have already been dispatched. + * + * If a hash_lock needs a dedupe context, and the available list is empty, the timed_out list will + * be searched for any contexts which are timed out and complete. One of these will be used + * immediately, and the rest will be returned to the available list and marked idle. + */ + +#include "dedupe.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "permassert.h" +#include "string-utils.h" + +#include "indexer.h" + +#include "action-manager.h" +#include "admin-state.h" +#include "completion.h" +#include "constants.h" +#include "data-vio.h" +#include "int-map.h" +#include "io-submitter.h" +#include "packer.h" +#include "physical-zone.h" +#include "slab-depot.h" +#include "statistics.h" +#include "types.h" +#include "vdo.h" +#include "wait-queue.h" + +struct uds_attribute { + struct attribute attr; + const char *(*show_string)(struct hash_zones *hash_zones); +}; + +#define DEDUPE_QUERY_TIMER_IDLE 0 +#define DEDUPE_QUERY_TIMER_RUNNING 1 +#define DEDUPE_QUERY_TIMER_FIRED 2 + +enum dedupe_context_state { + DEDUPE_CONTEXT_IDLE, + DEDUPE_CONTEXT_PENDING, + DEDUPE_CONTEXT_TIMED_OUT, + DEDUPE_CONTEXT_COMPLETE, + DEDUPE_CONTEXT_TIMED_OUT_COMPLETE, +}; + +/* Possible index states: closed, opened, or transitioning between those two. */ +enum index_state { + IS_CLOSED, + IS_CHANGING, + IS_OPENED, +}; + +static const char *CLOSED = "closed"; +static const char *CLOSING = "closing"; +static const char *ERROR = "error"; +static const char *OFFLINE = "offline"; +static const char *ONLINE = "online"; +static const char *OPENING = "opening"; +static const char *SUSPENDED = "suspended"; +static const char *UNKNOWN = "unknown"; + +/* Version 2 uses the kernel space UDS index and is limited to 16 bytes */ +#define UDS_ADVICE_VERSION 2 +/* version byte + state byte + 64-bit little-endian PBN */ +#define UDS_ADVICE_SIZE (1 + 1 + sizeof(u64)) + +enum hash_lock_state { + /* State for locks that are not in use or are being initialized. */ + VDO_HASH_LOCK_INITIALIZING, + + /* This is the sequence of states typically used on the non-dedupe path. */ + VDO_HASH_LOCK_QUERYING, + VDO_HASH_LOCK_WRITING, + VDO_HASH_LOCK_UPDATING, + + /* The remaining states are typically used on the dedupe path in this order. */ + VDO_HASH_LOCK_LOCKING, + VDO_HASH_LOCK_VERIFYING, + VDO_HASH_LOCK_DEDUPING, + VDO_HASH_LOCK_UNLOCKING, + + /* + * Terminal state for locks returning to the pool. Must be last both because it's the final + * state, and also because it's used to count the states. + */ + VDO_HASH_LOCK_BYPASSING, +}; + +static const char * const LOCK_STATE_NAMES[] = { + [VDO_HASH_LOCK_BYPASSING] = "BYPASSING", + [VDO_HASH_LOCK_DEDUPING] = "DEDUPING", + [VDO_HASH_LOCK_INITIALIZING] = "INITIALIZING", + [VDO_HASH_LOCK_LOCKING] = "LOCKING", + [VDO_HASH_LOCK_QUERYING] = "QUERYING", + [VDO_HASH_LOCK_UNLOCKING] = "UNLOCKING", + [VDO_HASH_LOCK_UPDATING] = "UPDATING", + [VDO_HASH_LOCK_VERIFYING] = "VERIFYING", + [VDO_HASH_LOCK_WRITING] = "WRITING", +}; + +struct hash_lock { + /* The block hash covered by this lock */ + struct uds_record_name hash; + + /* When the lock is unused, this list entry allows the lock to be pooled */ + struct list_head pool_node; + + /* + * A list containing the data VIOs sharing this lock, all having the same record name and + * data block contents, linked by their hash_lock_node fields. + */ + struct list_head duplicate_ring; + + /* The number of data_vios sharing this lock instance */ + data_vio_count_t reference_count; + + /* The maximum value of reference_count in the lifetime of this lock */ + data_vio_count_t max_references; + + /* The current state of this lock */ + enum hash_lock_state state; + + /* True if the UDS index should be updated with new advice */ + bool update_advice; + + /* True if the advice has been verified to be a true duplicate */ + bool verified; + + /* True if the lock has already accounted for an initial verification */ + bool verify_counted; + + /* True if this lock is registered in the lock map (cleared on rollover) */ + bool registered; + + /* + * If verified is false, this is the location of a possible duplicate. If verified is true, + * it is the verified location of a true duplicate. + */ + struct zoned_pbn duplicate; + + /* The PBN lock on the block containing the duplicate data */ + struct pbn_lock *duplicate_lock; + + /* The data_vio designated to act on behalf of the lock */ + struct data_vio *agent; + + /* + * Other data_vios with data identical to the agent who are currently waiting for the agent + * to get the information they all need to deduplicate--either against each other, or + * against an existing duplicate on disk. + */ + struct vdo_wait_queue waiters; +}; + +#define LOCK_POOL_CAPACITY MAXIMUM_VDO_USER_VIOS + +struct hash_zones { + struct action_manager *manager; + struct uds_parameters parameters; + struct uds_index_session *index_session; + struct ratelimit_state ratelimiter; + atomic64_t timeouts; + atomic64_t dedupe_context_busy; + + /* This spinlock protects the state fields and the starting of dedupe requests. */ + spinlock_t lock; + + /* The fields in the next block are all protected by the lock */ + struct vdo_completion completion; + enum index_state index_state; + enum index_state index_target; + struct admin_state state; + bool changing; + bool create_flag; + bool dedupe_flag; + bool error_flag; + u64 reported_timeouts; + + /* The number of zones */ + zone_count_t zone_count; + /* The hash zones themselves */ + struct hash_zone zones[]; +}; + +/* These are in milliseconds. */ +unsigned int vdo_dedupe_index_timeout_interval = 5000; +unsigned int vdo_dedupe_index_min_timer_interval = 100; +/* Same two variables, in jiffies for easier consumption. */ +static u64 vdo_dedupe_index_timeout_jiffies; +static u64 vdo_dedupe_index_min_timer_jiffies; + +static inline struct hash_zone *as_hash_zone(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_HASH_ZONE_COMPLETION); + return container_of(completion, struct hash_zone, completion); +} + +static inline struct hash_zones *as_hash_zones(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_HASH_ZONES_COMPLETION); + return container_of(completion, struct hash_zones, completion); +} + +static inline void assert_in_hash_zone(struct hash_zone *zone, const char *name) +{ + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == zone->thread_id), + "%s called on hash zone thread", name); +} + +static inline bool change_context_state(struct dedupe_context *context, int old, int new) +{ + return (atomic_cmpxchg(&context->state, old, new) == old); +} + +static inline bool change_timer_state(struct hash_zone *zone, int old, int new) +{ + return (atomic_cmpxchg(&zone->timer_state, old, new) == old); +} + +/** + * return_hash_lock_to_pool() - (Re)initialize a hash lock and return it to its pool. + * @zone: The zone from which the lock was borrowed. + * @lock: The lock that is no longer in use. + */ +static void return_hash_lock_to_pool(struct hash_zone *zone, struct hash_lock *lock) +{ + memset(lock, 0, sizeof(*lock)); + INIT_LIST_HEAD(&lock->pool_node); + INIT_LIST_HEAD(&lock->duplicate_ring); + vdo_waitq_init(&lock->waiters); + list_add_tail(&lock->pool_node, &zone->lock_pool); +} + +/** + * vdo_get_duplicate_lock() - Get the PBN lock on the duplicate data location for a data_vio from + * the hash_lock the data_vio holds (if there is one). + * @data_vio: The data_vio to query. + * + * Return: The PBN lock on the data_vio's duplicate location. + */ +struct pbn_lock *vdo_get_duplicate_lock(struct data_vio *data_vio) +{ + if (data_vio->hash_lock == NULL) + return NULL; + + return data_vio->hash_lock->duplicate_lock; +} + +/** + * hash_lock_key() - Return hash_lock's record name as a hash code. + * @lock: The hash lock. + * + * Return: The key to use for the int map. + */ +static inline u64 hash_lock_key(struct hash_lock *lock) +{ + return get_unaligned_le64(&lock->hash.name); +} + +/** + * get_hash_lock_state_name() - Get the string representation of a hash lock state. + * @state: The hash lock state. + * + * Return: The short string representing the state + */ +static const char *get_hash_lock_state_name(enum hash_lock_state state) +{ + /* Catch if a state has been added without updating the name array. */ + BUILD_BUG_ON((VDO_HASH_LOCK_BYPASSING + 1) != ARRAY_SIZE(LOCK_STATE_NAMES)); + return (state < ARRAY_SIZE(LOCK_STATE_NAMES)) ? LOCK_STATE_NAMES[state] : "INVALID"; +} + +/** + * assert_hash_lock_agent() - Assert that a data_vio is the agent of its hash lock, and that this + * is being called in the hash zone. + * @data_vio: The data_vio expected to be the lock agent. + * @where: A string describing the function making the assertion. + */ +static void assert_hash_lock_agent(struct data_vio *data_vio, const char *where) +{ + /* Not safe to access the agent field except from the hash zone. */ + assert_data_vio_in_hash_zone(data_vio); + VDO_ASSERT_LOG_ONLY(data_vio == data_vio->hash_lock->agent, + "%s must be for the hash lock agent", where); +} + +/** + * set_duplicate_lock() - Set the duplicate lock held by a hash lock. May only be called in the + * physical zone of the PBN lock. + * @hash_lock: The hash lock to update. + * @pbn_lock: The PBN read lock to use as the duplicate lock. + */ +static void set_duplicate_lock(struct hash_lock *hash_lock, struct pbn_lock *pbn_lock) +{ + VDO_ASSERT_LOG_ONLY((hash_lock->duplicate_lock == NULL), + "hash lock must not already hold a duplicate lock"); + pbn_lock->holder_count += 1; + hash_lock->duplicate_lock = pbn_lock; +} + +/** + * dequeue_lock_waiter() - Remove the first data_vio from the lock's waitq and return it. + * @lock: The lock containing the wait queue. + * + * Return: The first (oldest) waiter in the queue, or NULL if the queue is empty. + */ +static inline struct data_vio *dequeue_lock_waiter(struct hash_lock *lock) +{ + return vdo_waiter_as_data_vio(vdo_waitq_dequeue_waiter(&lock->waiters)); +} + +/** + * set_hash_lock() - Set, change, or clear the hash lock a data_vio is using. + * @data_vio: The data_vio to update. + * @new_lock: The hash lock the data_vio is joining. + * + * Updates the hash lock (or locks) to reflect the change in membership. + */ +static void set_hash_lock(struct data_vio *data_vio, struct hash_lock *new_lock) +{ + struct hash_lock *old_lock = data_vio->hash_lock; + + if (old_lock != NULL) { + VDO_ASSERT_LOG_ONLY(data_vio->hash_zone != NULL, + "must have a hash zone when holding a hash lock"); + VDO_ASSERT_LOG_ONLY(!list_empty(&data_vio->hash_lock_entry), + "must be on a hash lock ring when holding a hash lock"); + VDO_ASSERT_LOG_ONLY(old_lock->reference_count > 0, + "hash lock reference must be counted"); + + if ((old_lock->state != VDO_HASH_LOCK_BYPASSING) && + (old_lock->state != VDO_HASH_LOCK_UNLOCKING)) { + /* + * If the reference count goes to zero in a non-terminal state, we're most + * likely leaking this lock. + */ + VDO_ASSERT_LOG_ONLY(old_lock->reference_count > 1, + "hash locks should only become unreferenced in a terminal state, not state %s", + get_hash_lock_state_name(old_lock->state)); + } + + list_del_init(&data_vio->hash_lock_entry); + old_lock->reference_count -= 1; + + data_vio->hash_lock = NULL; + } + + if (new_lock != NULL) { + /* + * Keep all data_vios sharing the lock on a ring since they can complete in any + * order and we'll always need a pointer to one to compare data. + */ + list_move_tail(&data_vio->hash_lock_entry, &new_lock->duplicate_ring); + new_lock->reference_count += 1; + if (new_lock->max_references < new_lock->reference_count) + new_lock->max_references = new_lock->reference_count; + + data_vio->hash_lock = new_lock; + } +} + +/* There are loops in the state diagram, so some forward decl's are needed. */ +static void start_deduping(struct hash_lock *lock, struct data_vio *agent, + bool agent_is_done); +static void start_locking(struct hash_lock *lock, struct data_vio *agent); +static void start_writing(struct hash_lock *lock, struct data_vio *agent); +static void unlock_duplicate_pbn(struct vdo_completion *completion); +static void transfer_allocation_lock(struct data_vio *data_vio); + +/** + * exit_hash_lock() - Bottleneck for data_vios that have written or deduplicated and that are no + * longer needed to be an agent for the hash lock. + * @data_vio: The data_vio to complete and send to be cleaned up. + */ +static void exit_hash_lock(struct data_vio *data_vio) +{ + /* Release the hash lock now, saving a thread transition in cleanup. */ + vdo_release_hash_lock(data_vio); + + /* Complete the data_vio and start the clean-up path to release any locks it still holds. */ + data_vio->vio.completion.callback = complete_data_vio; + + continue_data_vio(data_vio); +} + +/** + * set_duplicate_location() - Set the location of the duplicate block for data_vio, updating the + * is_duplicate and duplicate fields from a zoned_pbn. + * @data_vio: The data_vio to modify. + * @source: The location of the duplicate. + */ +static void set_duplicate_location(struct data_vio *data_vio, + const struct zoned_pbn source) +{ + data_vio->is_duplicate = (source.pbn != VDO_ZERO_BLOCK); + data_vio->duplicate = source; +} + +/** + * retire_lock_agent() - Retire the active lock agent, replacing it with the first lock waiter, and + * make the retired agent exit the hash lock. + * @lock: The hash lock to update. + * + * Return: The new lock agent (which will be NULL if there was no waiter) + */ +static struct data_vio *retire_lock_agent(struct hash_lock *lock) +{ + struct data_vio *old_agent = lock->agent; + struct data_vio *new_agent = dequeue_lock_waiter(lock); + + lock->agent = new_agent; + exit_hash_lock(old_agent); + if (new_agent != NULL) + set_duplicate_location(new_agent, lock->duplicate); + return new_agent; +} + +/** + * wait_on_hash_lock() - Add a data_vio to the lock's queue of waiters. + * @lock: The hash lock on which to wait. + * @data_vio: The data_vio to add to the queue. + */ +static void wait_on_hash_lock(struct hash_lock *lock, struct data_vio *data_vio) +{ + vdo_waitq_enqueue_waiter(&lock->waiters, &data_vio->waiter); + + /* + * Make sure the agent doesn't block indefinitely in the packer since it now has at least + * one other data_vio waiting on it. + */ + if ((lock->state != VDO_HASH_LOCK_WRITING) || !cancel_data_vio_compression(lock->agent)) + return; + + /* + * Even though we're waiting, we also have to send ourselves as a one-way message to the + * packer to ensure the agent continues executing. This is safe because + * cancel_vio_compression() guarantees the agent won't continue executing until this + * message arrives in the packer, and because the wait queue link isn't used for sending + * the message. + */ + data_vio->compression.lock_holder = lock->agent; + launch_data_vio_packer_callback(data_vio, vdo_remove_lock_holder_from_packer); +} + +/** + * abort_waiter() - waiter_callback_fn function that shunts waiters to write their blocks without + * optimization. + * @waiter: The data_vio's waiter link. + * @context: Not used. + */ +static void abort_waiter(struct vdo_waiter *waiter, void *context __always_unused) +{ + write_data_vio(vdo_waiter_as_data_vio(waiter)); +} + +/** + * start_bypassing() - Stop using the hash lock. + * @lock: The hash lock. + * @agent: The data_vio acting as the agent for the lock. + * + * Stops using the hash lock. This is the final transition for hash locks which did not get an + * error. + */ +static void start_bypassing(struct hash_lock *lock, struct data_vio *agent) +{ + lock->state = VDO_HASH_LOCK_BYPASSING; + exit_hash_lock(agent); +} + +void vdo_clean_failed_hash_lock(struct data_vio *data_vio) +{ + struct hash_lock *lock = data_vio->hash_lock; + + if (lock->state == VDO_HASH_LOCK_BYPASSING) { + exit_hash_lock(data_vio); + return; + } + + if (lock->agent == NULL) { + lock->agent = data_vio; + } else if (data_vio != lock->agent) { + exit_hash_lock(data_vio); + return; + } + + lock->state = VDO_HASH_LOCK_BYPASSING; + + /* Ensure we don't attempt to update advice when cleaning up. */ + lock->update_advice = false; + + vdo_waitq_notify_all_waiters(&lock->waiters, abort_waiter, NULL); + + if (lock->duplicate_lock != NULL) { + /* The agent must reference the duplicate zone to launch it. */ + data_vio->duplicate = lock->duplicate; + launch_data_vio_duplicate_zone_callback(data_vio, unlock_duplicate_pbn); + return; + } + + lock->agent = NULL; + data_vio->is_duplicate = false; + exit_hash_lock(data_vio); +} + +/** + * finish_unlocking() - Handle the result of the agent for the lock releasing a read lock on + * duplicate candidate. + * @completion: The completion of the data_vio acting as the lock's agent. + * + * This continuation is registered in unlock_duplicate_pbn(). + */ +static void finish_unlocking(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + struct hash_lock *lock = agent->hash_lock; + + assert_hash_lock_agent(agent, __func__); + + VDO_ASSERT_LOG_ONLY(lock->duplicate_lock == NULL, + "must have released the duplicate lock for the hash lock"); + + if (!lock->verified) { + /* + * UNLOCKING -> WRITING transition: The lock we released was on an unverified + * block, so it must have been a lock on advice we were verifying, not on a + * location that was used for deduplication. Go write (or compress) the block to + * get a location to dedupe against. + */ + start_writing(lock, agent); + return; + } + + /* + * With the lock released, the verified duplicate block may already have changed and will + * need to be re-verified if a waiter arrived. + */ + lock->verified = false; + + if (vdo_waitq_has_waiters(&lock->waiters)) { + /* + * UNLOCKING -> LOCKING transition: A new data_vio entered the hash lock while the + * agent was releasing the PBN lock. The current agent exits and the waiter has to + * re-lock and re-verify the duplicate location. + * + * TODO: If we used the current agent to re-acquire the PBN lock we wouldn't need + * to re-verify. + */ + agent = retire_lock_agent(lock); + start_locking(lock, agent); + return; + } + + /* + * UNLOCKING -> BYPASSING transition: The agent is done with the lock and no other + * data_vios reference it, so remove it from the lock map and return it to the pool. + */ + start_bypassing(lock, agent); +} + +/** + * unlock_duplicate_pbn() - Release a read lock on the PBN of the block that may or may not have + * contained duplicate data. + * @completion: The completion of the data_vio acting as the lock's agent. + * + * This continuation is launched by start_unlocking(), and calls back to finish_unlocking() on the + * hash zone thread. + */ +static void unlock_duplicate_pbn(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + struct hash_lock *lock = agent->hash_lock; + + assert_data_vio_in_duplicate_zone(agent); + VDO_ASSERT_LOG_ONLY(lock->duplicate_lock != NULL, + "must have a duplicate lock to release"); + + vdo_release_physical_zone_pbn_lock(agent->duplicate.zone, agent->duplicate.pbn, + vdo_forget(lock->duplicate_lock)); + if (lock->state == VDO_HASH_LOCK_BYPASSING) { + complete_data_vio(completion); + return; + } + + launch_data_vio_hash_zone_callback(agent, finish_unlocking); +} + +/** + * start_unlocking() - Release a read lock on the PBN of the block that may or may not have + * contained duplicate data. + * @lock: The hash lock. + * @agent: The data_vio currently acting as the agent for the lock. + */ +static void start_unlocking(struct hash_lock *lock, struct data_vio *agent) +{ + lock->state = VDO_HASH_LOCK_UNLOCKING; + launch_data_vio_duplicate_zone_callback(agent, unlock_duplicate_pbn); +} + +static void release_context(struct dedupe_context *context) +{ + struct hash_zone *zone = context->zone; + + WRITE_ONCE(zone->active, zone->active - 1); + list_move(&context->list_entry, &zone->available); +} + +static void process_update_result(struct data_vio *agent) +{ + struct dedupe_context *context = agent->dedupe_context; + + if ((context == NULL) || + !change_context_state(context, DEDUPE_CONTEXT_COMPLETE, DEDUPE_CONTEXT_IDLE)) + return; + + release_context(context); +} + +/** + * finish_updating() - Process the result of a UDS update performed by the agent for the lock. + * @completion: The completion of the data_vio that performed the update + * + * This continuation is registered in start_querying(). + */ +static void finish_updating(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + struct hash_lock *lock = agent->hash_lock; + + assert_hash_lock_agent(agent, __func__); + + process_update_result(agent); + + /* + * UDS was updated successfully, so don't update again unless the duplicate location + * changes due to rollover. + */ + lock->update_advice = false; + + if (vdo_waitq_has_waiters(&lock->waiters)) { + /* + * UPDATING -> DEDUPING transition: A new data_vio arrived during the UDS update. + * Send it on the verified dedupe path. The agent is done with the lock, but the + * lock may still need to use it to clean up after rollover. + */ + start_deduping(lock, agent, true); + return; + } + + if (lock->duplicate_lock != NULL) { + /* + * UPDATING -> UNLOCKING transition: No one is waiting to dedupe, but we hold a + * duplicate PBN lock, so go release it. + */ + start_unlocking(lock, agent); + return; + } + + /* + * UPDATING -> BYPASSING transition: No one is waiting to dedupe and there's no lock to + * release. + */ + start_bypassing(lock, agent); +} + +static void query_index(struct data_vio *data_vio, enum uds_request_type operation); + +/** + * start_updating() - Continue deduplication with the last step, updating UDS with the location of + * the duplicate that should be returned as advice in the future. + * @lock: The hash lock. + * @agent: The data_vio currently acting as the agent for the lock. + */ +static void start_updating(struct hash_lock *lock, struct data_vio *agent) +{ + lock->state = VDO_HASH_LOCK_UPDATING; + + VDO_ASSERT_LOG_ONLY(lock->verified, "new advice should have been verified"); + VDO_ASSERT_LOG_ONLY(lock->update_advice, "should only update advice if needed"); + + agent->last_async_operation = VIO_ASYNC_OP_UPDATE_DEDUPE_INDEX; + set_data_vio_hash_zone_callback(agent, finish_updating); + query_index(agent, UDS_UPDATE); +} + +/** + * finish_deduping() - Handle a data_vio that has finished deduplicating against the block locked + * by the hash lock. + * @lock: The hash lock. + * @data_vio: The lock holder that has finished deduplicating. + * + * If there are other data_vios still sharing the lock, this will just release the data_vio's share + * of the lock and finish processing the data_vio. If this is the last data_vio holding the lock, + * this makes the data_vio the lock agent and uses it to advance the state of the lock so it can + * eventually be released. + */ +static void finish_deduping(struct hash_lock *lock, struct data_vio *data_vio) +{ + struct data_vio *agent = data_vio; + + VDO_ASSERT_LOG_ONLY(lock->agent == NULL, "shouldn't have an agent in DEDUPING"); + VDO_ASSERT_LOG_ONLY(!vdo_waitq_has_waiters(&lock->waiters), + "shouldn't have any lock waiters in DEDUPING"); + + /* Just release the lock reference if other data_vios are still deduping. */ + if (lock->reference_count > 1) { + exit_hash_lock(data_vio); + return; + } + + /* The hash lock must have an agent for all other lock states. */ + lock->agent = agent; + if (lock->update_advice) { + /* + * DEDUPING -> UPDATING transition: The location of the duplicate block changed + * since the initial UDS query because of compression, rollover, or because the + * query agent didn't have an allocation. The UDS update was delayed in case there + * was another change in location, but with only this data_vio using the hash lock, + * it's time to update the advice. + */ + start_updating(lock, agent); + } else { + /* + * DEDUPING -> UNLOCKING transition: Release the PBN read lock on the duplicate + * location so the hash lock itself can be released (contingent on no new data_vios + * arriving in the lock before the agent returns). + */ + start_unlocking(lock, agent); + } +} + +/** + * acquire_lock() - Get the lock for a record name. + * @zone: The zone responsible for the hash. + * @hash: The hash to lock. + * @replace_lock: If non-NULL, the lock already registered for the hash which should be replaced by + * the new lock. + * @lock_ptr: A pointer to receive the hash lock. + * + * Gets the lock for the hash (record name) of the data in a data_vio, or if one does not exist (or + * if we are explicitly rolling over), initialize a new lock for the hash and register it in the + * zone. This must only be called in the correct thread for the zone. + * + * Return: VDO_SUCCESS or an error code. + */ +static int __must_check acquire_lock(struct hash_zone *zone, + const struct uds_record_name *hash, + struct hash_lock *replace_lock, + struct hash_lock **lock_ptr) +{ + struct hash_lock *lock, *new_lock; + int result; + + /* + * Borrow and prepare a lock from the pool so we don't have to do two int_map accesses + * in the common case of no lock contention. + */ + result = VDO_ASSERT(!list_empty(&zone->lock_pool), + "never need to wait for a free hash lock"); + if (result != VDO_SUCCESS) + return result; + + new_lock = list_entry(zone->lock_pool.prev, struct hash_lock, pool_node); + list_del_init(&new_lock->pool_node); + + /* + * Fill in the hash of the new lock so we can map it, since we have to use the hash as the + * map key. + */ + new_lock->hash = *hash; + + result = vdo_int_map_put(zone->hash_lock_map, hash_lock_key(new_lock), + new_lock, (replace_lock != NULL), (void **) &lock); + if (result != VDO_SUCCESS) { + return_hash_lock_to_pool(zone, vdo_forget(new_lock)); + return result; + } + + if (replace_lock != NULL) { + /* On mismatch put the old lock back and return a severe error */ + VDO_ASSERT_LOG_ONLY(lock == replace_lock, + "old lock must have been in the lock map"); + /* TODO: Check earlier and bail out? */ + VDO_ASSERT_LOG_ONLY(replace_lock->registered, + "old lock must have been marked registered"); + replace_lock->registered = false; + } + + if (lock == replace_lock) { + lock = new_lock; + lock->registered = true; + } else { + /* There's already a lock for the hash, so we don't need the borrowed lock. */ + return_hash_lock_to_pool(zone, vdo_forget(new_lock)); + } + + *lock_ptr = lock; + return VDO_SUCCESS; +} + +/** + * enter_forked_lock() - Bind the data_vio to a new hash lock. + * + * Implements waiter_callback_fn. Binds the data_vio that was waiting to a new hash lock and waits + * on that lock. + */ +static void enter_forked_lock(struct vdo_waiter *waiter, void *context) +{ + struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); + struct hash_lock *new_lock = context; + + set_hash_lock(data_vio, new_lock); + wait_on_hash_lock(new_lock, data_vio); +} + +/** + * fork_hash_lock() - Fork a hash lock because it has run out of increments on the duplicate PBN. + * @old_lock: The hash lock to fork. + * @new_agent: The data_vio that will be the agent for the new lock. + * + * Transfers the new agent and any lock waiters to a new hash lock instance which takes the place + * of the old lock in the lock map. The old lock remains active, but will not update advice. + */ +static void fork_hash_lock(struct hash_lock *old_lock, struct data_vio *new_agent) +{ + struct hash_lock *new_lock; + int result; + + result = acquire_lock(new_agent->hash_zone, &new_agent->record_name, old_lock, + &new_lock); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(new_agent, result); + return; + } + + /* + * Only one of the two locks should update UDS. The old lock is out of references, so it + * would be poor dedupe advice in the short term. + */ + old_lock->update_advice = false; + new_lock->update_advice = true; + + set_hash_lock(new_agent, new_lock); + new_lock->agent = new_agent; + + vdo_waitq_notify_all_waiters(&old_lock->waiters, enter_forked_lock, new_lock); + + new_agent->is_duplicate = false; + start_writing(new_lock, new_agent); +} + +/** + * launch_dedupe() - Reserve a reference count increment for a data_vio and launch it on the dedupe + * path. + * @lock: The hash lock. + * @data_vio: The data_vio to deduplicate using the hash lock. + * @has_claim: true if the data_vio already has claimed an increment from the duplicate lock. + * + * If no increments are available, this will roll over to a new hash lock and launch the data_vio + * as the writing agent for that lock. + */ +static void launch_dedupe(struct hash_lock *lock, struct data_vio *data_vio, + bool has_claim) +{ + if (!has_claim && !vdo_claim_pbn_lock_increment(lock->duplicate_lock)) { + /* Out of increments, so must roll over to a new lock. */ + fork_hash_lock(lock, data_vio); + return; + } + + /* Deduplicate against the lock's verified location. */ + set_duplicate_location(data_vio, lock->duplicate); + data_vio->new_mapped = data_vio->duplicate; + update_metadata_for_data_vio_write(data_vio, lock->duplicate_lock); +} + +/** + * start_deduping() - Enter the hash lock state where data_vios deduplicate in parallel against a + * true copy of their data on disk. + * @lock: The hash lock. + * @agent: The data_vio acting as the agent for the lock. + * @agent_is_done: true only if the agent has already written or deduplicated against its data. + * + * If the agent itself needs to deduplicate, an increment for it must already have been claimed + * from the duplicate lock, ensuring the hash lock will still have a data_vio holding it. + */ +static void start_deduping(struct hash_lock *lock, struct data_vio *agent, + bool agent_is_done) +{ + lock->state = VDO_HASH_LOCK_DEDUPING; + + /* + * We don't take the downgraded allocation lock from the agent unless we actually need to + * deduplicate against it. + */ + if (lock->duplicate_lock == NULL) { + VDO_ASSERT_LOG_ONLY(!vdo_is_state_compressed(agent->new_mapped.state), + "compression must have shared a lock"); + VDO_ASSERT_LOG_ONLY(agent_is_done, + "agent must have written the new duplicate"); + transfer_allocation_lock(agent); + } + + VDO_ASSERT_LOG_ONLY(vdo_is_pbn_read_lock(lock->duplicate_lock), + "duplicate_lock must be a PBN read lock"); + + /* + * This state is not like any of the other states. There is no designated agent--the agent + * transitioning to this state and all the waiters will be launched to deduplicate in + * parallel. + */ + lock->agent = NULL; + + /* + * Launch the agent (if not already deduplicated) and as many lock waiters as we have + * available increments for on the dedupe path. If we run out of increments, rollover will + * be triggered and the remaining waiters will be transferred to the new lock. + */ + if (!agent_is_done) { + launch_dedupe(lock, agent, true); + agent = NULL; + } + while (vdo_waitq_has_waiters(&lock->waiters)) + launch_dedupe(lock, dequeue_lock_waiter(lock), false); + + if (agent_is_done) { + /* + * In the degenerate case where all the waiters rolled over to a new lock, this + * will continue to use the old agent to clean up this lock, and otherwise it just + * lets the agent exit the lock. + */ + finish_deduping(lock, agent); + } +} + +/** + * increment_stat() - Increment a statistic counter in a non-atomic yet thread-safe manner. + * @stat: The statistic field to increment. + */ +static inline void increment_stat(u64 *stat) +{ + /* + * Must only be mutated on the hash zone thread. Prevents any compiler shenanigans from + * affecting other threads reading stats. + */ + WRITE_ONCE(*stat, *stat + 1); +} + +/** + * finish_verifying() - Handle the result of the agent for the lock comparing its data to the + * duplicate candidate. + * @completion: The completion of the data_vio used to verify dedupe + * + * This continuation is registered in start_verifying(). + */ +static void finish_verifying(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + struct hash_lock *lock = agent->hash_lock; + + assert_hash_lock_agent(agent, __func__); + + lock->verified = agent->is_duplicate; + + /* + * Only count the result of the initial verification of the advice as valid or stale, and + * not any re-verifications due to PBN lock releases. + */ + if (!lock->verify_counted) { + lock->verify_counted = true; + if (lock->verified) + increment_stat(&agent->hash_zone->statistics.dedupe_advice_valid); + else + increment_stat(&agent->hash_zone->statistics.dedupe_advice_stale); + } + + /* + * Even if the block is a verified duplicate, we can't start to deduplicate unless we can + * claim a reference count increment for the agent. + */ + if (lock->verified && !vdo_claim_pbn_lock_increment(lock->duplicate_lock)) { + agent->is_duplicate = false; + lock->verified = false; + } + + if (lock->verified) { + /* + * VERIFYING -> DEDUPING transition: The advice is for a true duplicate, so start + * deduplicating against it, if references are available. + */ + start_deduping(lock, agent, false); + } else { + /* + * VERIFYING -> UNLOCKING transition: Either the verify failed or we'd try to + * dedupe and roll over immediately, which would fail because it would leave the + * lock without an agent to release the PBN lock. In both cases, the data will have + * to be written or compressed, but first the advice PBN must be unlocked by the + * VERIFYING agent. + */ + lock->update_advice = true; + start_unlocking(lock, agent); + } +} + +static bool blocks_equal(char *block1, char *block2) +{ + int i; + + for (i = 0; i < VDO_BLOCK_SIZE; i += sizeof(u64)) { + if (*((u64 *) &block1[i]) != *((u64 *) &block2[i])) + return false; + } + + return true; +} + +static void verify_callback(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + + agent->is_duplicate = blocks_equal(agent->vio.data, agent->scratch_block); + launch_data_vio_hash_zone_callback(agent, finish_verifying); +} + +static void uncompress_and_verify(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + int result; + + result = uncompress_data_vio(agent, agent->duplicate.state, + agent->scratch_block); + if (result == VDO_SUCCESS) { + verify_callback(completion); + return; + } + + agent->is_duplicate = false; + launch_data_vio_hash_zone_callback(agent, finish_verifying); +} + +static void verify_endio(struct bio *bio) +{ + struct data_vio *agent = vio_as_data_vio(bio->bi_private); + int result = blk_status_to_errno(bio->bi_status); + + vdo_count_completed_bios(bio); + if (result != VDO_SUCCESS) { + agent->is_duplicate = false; + launch_data_vio_hash_zone_callback(agent, finish_verifying); + return; + } + + if (vdo_is_state_compressed(agent->duplicate.state)) { + launch_data_vio_cpu_callback(agent, uncompress_and_verify, + CPU_Q_COMPRESS_BLOCK_PRIORITY); + return; + } + + launch_data_vio_cpu_callback(agent, verify_callback, + CPU_Q_COMPLETE_READ_PRIORITY); +} + +/** + * start_verifying() - Begin the data verification phase. + * @lock: The hash lock (must be LOCKING). + * @agent: The data_vio to use to read and compare candidate data. + * + * Continue the deduplication path for a hash lock by using the agent to read (and possibly + * decompress) the data at the candidate duplicate location, comparing it to the data in the agent + * to verify that the candidate is identical to all the data_vios sharing the hash. If so, it can + * be deduplicated against, otherwise a data_vio allocation will have to be written to and used for + * dedupe. + */ +static void start_verifying(struct hash_lock *lock, struct data_vio *agent) +{ + int result; + struct vio *vio = &agent->vio; + char *buffer = (vdo_is_state_compressed(agent->duplicate.state) ? + (char *) agent->compression.block : + agent->scratch_block); + + lock->state = VDO_HASH_LOCK_VERIFYING; + VDO_ASSERT_LOG_ONLY(!lock->verified, "hash lock only verifies advice once"); + + agent->last_async_operation = VIO_ASYNC_OP_VERIFY_DUPLICATION; + result = vio_reset_bio(vio, buffer, verify_endio, REQ_OP_READ, + agent->duplicate.pbn); + if (result != VDO_SUCCESS) { + set_data_vio_hash_zone_callback(agent, finish_verifying); + continue_data_vio_with_error(agent, result); + return; + } + + set_data_vio_bio_zone_callback(agent, vdo_submit_vio); + vdo_launch_completion_with_priority(&vio->completion, BIO_Q_VERIFY_PRIORITY); +} + +/** + * finish_locking() - Handle the result of the agent for the lock attempting to obtain a PBN read + * lock on the candidate duplicate block. + * @completion: The completion of the data_vio that attempted to get the read lock. + * + * This continuation is registered in lock_duplicate_pbn(). + */ +static void finish_locking(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + struct hash_lock *lock = agent->hash_lock; + + assert_hash_lock_agent(agent, __func__); + + if (!agent->is_duplicate) { + VDO_ASSERT_LOG_ONLY(lock->duplicate_lock == NULL, + "must not hold duplicate_lock if not flagged as a duplicate"); + /* + * LOCKING -> WRITING transition: The advice block is being modified or has no + * available references, so try to write or compress the data, remembering to + * update UDS later with the new advice. + */ + increment_stat(&agent->hash_zone->statistics.dedupe_advice_stale); + lock->update_advice = true; + start_writing(lock, agent); + return; + } + + VDO_ASSERT_LOG_ONLY(lock->duplicate_lock != NULL, + "must hold duplicate_lock if flagged as a duplicate"); + + if (!lock->verified) { + /* + * LOCKING -> VERIFYING transition: Continue on the unverified dedupe path, reading + * the candidate duplicate and comparing it to the agent's data to decide whether + * it is a true duplicate or stale advice. + */ + start_verifying(lock, agent); + return; + } + + if (!vdo_claim_pbn_lock_increment(lock->duplicate_lock)) { + /* + * LOCKING -> UNLOCKING transition: The verified block was re-locked, but has no + * available increments left. Must first release the useless PBN read lock before + * rolling over to a new copy of the block. + */ + agent->is_duplicate = false; + lock->verified = false; + lock->update_advice = true; + start_unlocking(lock, agent); + return; + } + + /* + * LOCKING -> DEDUPING transition: Continue on the verified dedupe path, deduplicating + * against a location that was previously verified or written to. + */ + start_deduping(lock, agent, false); +} + +static bool acquire_provisional_reference(struct data_vio *agent, struct pbn_lock *lock, + struct slab_depot *depot) +{ + /* Ensure that the newly-locked block is referenced. */ + struct vdo_slab *slab = vdo_get_slab(depot, agent->duplicate.pbn); + int result = vdo_acquire_provisional_reference(slab, agent->duplicate.pbn, lock); + + if (result == VDO_SUCCESS) + return true; + + vdo_log_warning_strerror(result, + "Error acquiring provisional reference for dedupe candidate; aborting dedupe"); + agent->is_duplicate = false; + vdo_release_physical_zone_pbn_lock(agent->duplicate.zone, + agent->duplicate.pbn, lock); + continue_data_vio_with_error(agent, result); + return false; +} + +/** + * lock_duplicate_pbn() - Acquire a read lock on the PBN of the block containing candidate + * duplicate data (compressed or uncompressed). + * @completion: The completion of the data_vio attempting to acquire the physical block lock on + * behalf of its hash lock. + * + * If the PBN is already locked for writing, the lock attempt is abandoned and is_duplicate will be + * cleared before calling back. This continuation is launched from start_locking(), and calls back + * to finish_locking() on the hash zone thread. + */ +static void lock_duplicate_pbn(struct vdo_completion *completion) +{ + unsigned int increment_limit; + struct pbn_lock *lock; + int result; + + struct data_vio *agent = as_data_vio(completion); + struct slab_depot *depot = vdo_from_data_vio(agent)->depot; + struct physical_zone *zone = agent->duplicate.zone; + + assert_data_vio_in_duplicate_zone(agent); + + set_data_vio_hash_zone_callback(agent, finish_locking); + + /* + * While in the zone that owns it, find out how many additional references can be made to + * the block if it turns out to truly be a duplicate. + */ + increment_limit = vdo_get_increment_limit(depot, agent->duplicate.pbn); + if (increment_limit == 0) { + /* + * We could deduplicate against it later if a reference happened to be released + * during verification, but it's probably better to bail out now. + */ + agent->is_duplicate = false; + continue_data_vio(agent); + return; + } + + result = vdo_attempt_physical_zone_pbn_lock(zone, agent->duplicate.pbn, + VIO_READ_LOCK, &lock); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(agent, result); + return; + } + + if (!vdo_is_pbn_read_lock(lock)) { + /* + * There are three cases of write locks: uncompressed data block writes, compressed + * (packed) block writes, and block map page writes. In all three cases, we give up + * on trying to verify the advice and don't bother to try deduplicate against the + * data in the write lock holder. + * + * 1) We don't ever want to try to deduplicate against a block map page. + * + * 2a) It's very unlikely we'd deduplicate against an entire packed block, both + * because of the chance of matching it, and because we don't record advice for it, + * but for the uncompressed representation of all the fragments it contains. The + * only way we'd be getting lock contention is if we've written the same + * representation coincidentally before, had it become unreferenced, and it just + * happened to be packed together from compressed writes when we go to verify the + * lucky advice. Giving up is a minuscule loss of potential dedupe. + * + * 2b) If the advice is for a slot of a compressed block, it's about to get + * smashed, and the write smashing it cannot contain our data--it would have to be + * writing on behalf of our hash lock, but that's impossible since we're the lock + * agent. + * + * 3a) If the lock is held by a data_vio with different data, the advice is already + * stale or is about to become stale. + * + * 3b) If the lock is held by a data_vio that matches us, we may as well either + * write it ourselves (or reference the copy we already wrote) instead of + * potentially having many duplicates wait for the lock holder to write, journal, + * hash, and finally arrive in the hash lock. We lose a chance to avoid a UDS + * update in the very rare case of advice for a free block that just happened to be + * allocated to a data_vio with the same hash. There's also a chance to save on a + * block write, at the cost of a block verify. Saving on a full block compare in + * all stale advice cases almost certainly outweighs saving a UDS update and + * trading a write for a read in a lucky case where advice would have been saved + * from becoming stale. + */ + agent->is_duplicate = false; + continue_data_vio(agent); + return; + } + + if (lock->holder_count == 0) { + if (!acquire_provisional_reference(agent, lock, depot)) + return; + + /* + * The increment limit we grabbed earlier is still valid. The lock now holds the + * rights to acquire all those references. Those rights will be claimed by hash + * locks sharing this read lock. + */ + lock->increment_limit = increment_limit; + } + + /* + * We've successfully acquired a read lock on behalf of the hash lock, so mark it as such. + */ + set_duplicate_lock(agent->hash_lock, lock); + + /* + * TODO: Optimization: We could directly launch the block verify, then switch to a hash + * thread. + */ + continue_data_vio(agent); +} + +/** + * start_locking() - Continue deduplication for a hash lock that has obtained valid advice of a + * potential duplicate through its agent. + * @lock: The hash lock (currently must be QUERYING). + * @agent: The data_vio bearing the dedupe advice. + */ +static void start_locking(struct hash_lock *lock, struct data_vio *agent) +{ + VDO_ASSERT_LOG_ONLY(lock->duplicate_lock == NULL, + "must not acquire a duplicate lock when already holding it"); + + lock->state = VDO_HASH_LOCK_LOCKING; + + /* + * TODO: Optimization: If we arrange to continue on the duplicate zone thread when + * accepting the advice, and don't explicitly change lock states (or use an agent-local + * state, or an atomic), we can avoid a thread transition here. + */ + agent->last_async_operation = VIO_ASYNC_OP_LOCK_DUPLICATE_PBN; + launch_data_vio_duplicate_zone_callback(agent, lock_duplicate_pbn); +} + +/** + * finish_writing() - Re-entry point for the lock agent after it has finished writing or + * compressing its copy of the data block. + * @lock: The hash lock, which must be in state WRITING. + * @agent: The data_vio that wrote its data for the lock. + * + * The agent will never need to dedupe against anything, so it's done with the lock, but the lock + * may not be finished with it, as a UDS update might still be needed. + * + * If there are other lock holders, the agent will hand the job to one of them and exit, leaving + * the lock to deduplicate against the just-written block. If there are no other lock holders, the + * agent either exits (and later tears down the hash lock), or it remains the agent and updates + * UDS. + */ +static void finish_writing(struct hash_lock *lock, struct data_vio *agent) +{ + /* + * Dedupe against the data block or compressed block slot the agent wrote. Since we know + * the write succeeded, there's no need to verify it. + */ + lock->duplicate = agent->new_mapped; + lock->verified = true; + + if (vdo_is_state_compressed(lock->duplicate.state) && lock->registered) { + /* + * Compression means the location we gave in the UDS query is not the location + * we're using to deduplicate. + */ + lock->update_advice = true; + } + + /* If there are any waiters, we need to start deduping them. */ + if (vdo_waitq_has_waiters(&lock->waiters)) { + /* + * WRITING -> DEDUPING transition: an asynchronously-written block failed to + * compress, so the PBN lock on the written copy was already transferred. The agent + * is done with the lock, but the lock may still need to use it to clean up after + * rollover. + */ + start_deduping(lock, agent, true); + return; + } + + /* + * There are no waiters and the agent has successfully written, so take a step towards + * being able to release the hash lock (or just release it). + */ + if (lock->update_advice) { + /* + * WRITING -> UPDATING transition: There's no waiter and a UDS update is needed, so + * retain the WRITING agent and use it to launch the update. The happens on + * compression, rollover, or the QUERYING agent not having an allocation. + */ + start_updating(lock, agent); + } else if (lock->duplicate_lock != NULL) { + /* + * WRITING -> UNLOCKING transition: There's no waiter and no update needed, but the + * compressed write gave us a shared duplicate lock that we must release. + */ + set_duplicate_location(agent, lock->duplicate); + start_unlocking(lock, agent); + } else { + /* + * WRITING -> BYPASSING transition: There's no waiter, no update needed, and no + * duplicate lock held, so both the agent and lock have no more work to do. The + * agent will release its allocation lock in cleanup. + */ + start_bypassing(lock, agent); + } +} + +/** + * select_writing_agent() - Search through the lock waiters for a data_vio that has an allocation. + * @lock: The hash lock to modify. + * + * If an allocation is found, swap agents, put the old agent at the head of the wait queue, then + * return the new agent. Otherwise, just return the current agent. + */ +static struct data_vio *select_writing_agent(struct hash_lock *lock) +{ + struct vdo_wait_queue temp_queue; + struct data_vio *data_vio; + + vdo_waitq_init(&temp_queue); + + /* + * Move waiters to the temp queue one-by-one until we find an allocation. Not ideal to + * search, but it only happens when nearly out of space. + */ + while (((data_vio = dequeue_lock_waiter(lock)) != NULL) && + !data_vio_has_allocation(data_vio)) { + /* Use the lower-level enqueue since we're just moving waiters around. */ + vdo_waitq_enqueue_waiter(&temp_queue, &data_vio->waiter); + } + + if (data_vio != NULL) { + /* + * Move the rest of the waiters over to the temp queue, preserving the order they + * arrived at the lock. + */ + vdo_waitq_transfer_all_waiters(&lock->waiters, &temp_queue); + + /* + * The current agent is being replaced and will have to wait to dedupe; make it the + * first waiter since it was the first to reach the lock. + */ + vdo_waitq_enqueue_waiter(&lock->waiters, &lock->agent->waiter); + lock->agent = data_vio; + } else { + /* No one has an allocation, so keep the current agent. */ + data_vio = lock->agent; + } + + /* Swap all the waiters back onto the lock's queue. */ + vdo_waitq_transfer_all_waiters(&temp_queue, &lock->waiters); + return data_vio; +} + +/** + * start_writing() - Begin the non-duplicate write path. + * @lock: The hash lock (currently must be QUERYING). + * @agent: The data_vio currently acting as the agent for the lock. + * + * Begins the non-duplicate write path for a hash lock that had no advice, selecting a data_vio + * with an allocation as a new agent, if necessary, then resuming the agent on the data_vio write + * path. + */ +static void start_writing(struct hash_lock *lock, struct data_vio *agent) +{ + lock->state = VDO_HASH_LOCK_WRITING; + + /* + * The agent might not have received an allocation and so can't be used for writing, but + * it's entirely possible that one of the waiters did. + */ + if (!data_vio_has_allocation(agent)) { + agent = select_writing_agent(lock); + /* If none of the waiters had an allocation, the writes all have to fail. */ + if (!data_vio_has_allocation(agent)) { + /* + * TODO: Should we keep a variant of BYPASSING that causes new arrivals to + * fail immediately if they don't have an allocation? It might be possible + * that on some path there would be non-waiters still referencing the lock, + * so it would remain in the map as everything is currently spelled, even + * if the agent and all waiters release. + */ + continue_data_vio_with_error(agent, VDO_NO_SPACE); + return; + } + } + + /* + * If the agent compresses, it might wait indefinitely in the packer, which would be bad if + * there are any other data_vios waiting. + */ + if (vdo_waitq_has_waiters(&lock->waiters)) + cancel_data_vio_compression(agent); + + /* + * Send the agent to the compress/pack/write path in vioWrite. If it succeeds, it will + * return to the hash lock via vdo_continue_hash_lock() and call finish_writing(). + */ + launch_compress_data_vio(agent); +} + +/* + * Decode VDO duplicate advice from the old_metadata field of a UDS request. + * Returns true if valid advice was found and decoded + */ +static bool decode_uds_advice(struct dedupe_context *context) +{ + const struct uds_request *request = &context->request; + struct data_vio *data_vio = context->requestor; + size_t offset = 0; + const struct uds_record_data *encoding = &request->old_metadata; + struct vdo *vdo = vdo_from_data_vio(data_vio); + struct zoned_pbn *advice = &data_vio->duplicate; + u8 version; + int result; + + if ((request->status != UDS_SUCCESS) || !request->found) + return false; + + version = encoding->data[offset++]; + if (version != UDS_ADVICE_VERSION) { + vdo_log_error("invalid UDS advice version code %u", version); + return false; + } + + advice->state = encoding->data[offset++]; + advice->pbn = get_unaligned_le64(&encoding->data[offset]); + offset += sizeof(u64); + BUG_ON(offset != UDS_ADVICE_SIZE); + + /* Don't use advice that's clearly meaningless. */ + if ((advice->state == VDO_MAPPING_STATE_UNMAPPED) || (advice->pbn == VDO_ZERO_BLOCK)) { + vdo_log_debug("Invalid advice from deduplication server: pbn %llu, state %u. Giving up on deduplication of logical block %llu", + (unsigned long long) advice->pbn, advice->state, + (unsigned long long) data_vio->logical.lbn); + atomic64_inc(&vdo->stats.invalid_advice_pbn_count); + return false; + } + + result = vdo_get_physical_zone(vdo, advice->pbn, &advice->zone); + if ((result != VDO_SUCCESS) || (advice->zone == NULL)) { + vdo_log_debug("Invalid physical block number from deduplication server: %llu, giving up on deduplication of logical block %llu", + (unsigned long long) advice->pbn, + (unsigned long long) data_vio->logical.lbn); + atomic64_inc(&vdo->stats.invalid_advice_pbn_count); + return false; + } + + return true; +} + +static void process_query_result(struct data_vio *agent) +{ + struct dedupe_context *context = agent->dedupe_context; + + if (context == NULL) + return; + + if (change_context_state(context, DEDUPE_CONTEXT_COMPLETE, DEDUPE_CONTEXT_IDLE)) { + agent->is_duplicate = decode_uds_advice(context); + release_context(context); + } +} + +/** + * finish_querying() - Process the result of a UDS query performed by the agent for the lock. + * @completion: The completion of the data_vio that performed the query. + * + * This continuation is registered in start_querying(). + */ +static void finish_querying(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + struct hash_lock *lock = agent->hash_lock; + + assert_hash_lock_agent(agent, __func__); + + process_query_result(agent); + + if (agent->is_duplicate) { + lock->duplicate = agent->duplicate; + /* + * QUERYING -> LOCKING transition: Valid advice was obtained from UDS. Use the + * QUERYING agent to start the hash lock on the unverified dedupe path, verifying + * that the advice can be used. + */ + start_locking(lock, agent); + } else { + /* + * The agent will be used as the duplicate if has an allocation; if it does, that + * location was posted to UDS, so no update will be needed. + */ + lock->update_advice = !data_vio_has_allocation(agent); + /* + * QUERYING -> WRITING transition: There was no advice or the advice wasn't valid, + * so try to write or compress the data. + */ + start_writing(lock, agent); + } +} + +/** + * start_querying() - Start deduplication for a hash lock. + * @lock: The initialized hash lock. + * @data_vio: The data_vio that has just obtained the new lock. + * + * Starts deduplication for a hash lock that has finished initializing by making the data_vio that + * requested it the agent, entering the QUERYING state, and using the agent to perform the UDS + * query on behalf of the lock. + */ +static void start_querying(struct hash_lock *lock, struct data_vio *data_vio) +{ + lock->agent = data_vio; + lock->state = VDO_HASH_LOCK_QUERYING; + data_vio->last_async_operation = VIO_ASYNC_OP_CHECK_FOR_DUPLICATION; + set_data_vio_hash_zone_callback(data_vio, finish_querying); + query_index(data_vio, + (data_vio_has_allocation(data_vio) ? UDS_POST : UDS_QUERY)); +} + +/** + * report_bogus_lock_state() - Complain that a data_vio has entered a hash_lock that is in an + * unimplemented or unusable state and continue the data_vio with an + * error. + * @lock: The hash lock. + * @data_vio: The data_vio attempting to enter the lock. + */ +static void report_bogus_lock_state(struct hash_lock *lock, struct data_vio *data_vio) +{ + VDO_ASSERT_LOG_ONLY(false, "hash lock must not be in unimplemented state %s", + get_hash_lock_state_name(lock->state)); + continue_data_vio_with_error(data_vio, VDO_LOCK_ERROR); +} + +/** + * vdo_continue_hash_lock() - Continue the processing state after writing, compressing, or + * deduplicating. + * @data_vio: The data_vio to continue processing in its hash lock. + * + * Asynchronously continue processing a data_vio in its hash lock after it has finished writing, + * compressing, or deduplicating, so it can share the result with any data_vios waiting in the hash + * lock, or update the UDS index, or simply release its share of the lock. + * + * Context: This must only be called in the correct thread for the hash zone. + */ +void vdo_continue_hash_lock(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + struct hash_lock *lock = data_vio->hash_lock; + + switch (lock->state) { + case VDO_HASH_LOCK_WRITING: + VDO_ASSERT_LOG_ONLY(data_vio == lock->agent, + "only the lock agent may continue the lock"); + finish_writing(lock, data_vio); + break; + + case VDO_HASH_LOCK_DEDUPING: + finish_deduping(lock, data_vio); + break; + + case VDO_HASH_LOCK_BYPASSING: + /* This data_vio has finished the write path and the lock doesn't need it. */ + exit_hash_lock(data_vio); + break; + + case VDO_HASH_LOCK_INITIALIZING: + case VDO_HASH_LOCK_QUERYING: + case VDO_HASH_LOCK_UPDATING: + case VDO_HASH_LOCK_LOCKING: + case VDO_HASH_LOCK_VERIFYING: + case VDO_HASH_LOCK_UNLOCKING: + /* A lock in this state should never be re-entered. */ + report_bogus_lock_state(lock, data_vio); + break; + + default: + report_bogus_lock_state(lock, data_vio); + } +} + +/** + * is_hash_collision() - Check to see if a hash collision has occurred. + * @lock: The lock to check. + * @candidate: The data_vio seeking to share the lock. + * + * Check whether the data in data_vios sharing a lock is different than in a data_vio seeking to + * share the lock, which should only be possible in the extremely unlikely case of a hash + * collision. + * + * Return: true if the given data_vio must not share the lock because it doesn't have the same data + * as the lock holders. + */ +static bool is_hash_collision(struct hash_lock *lock, struct data_vio *candidate) +{ + struct data_vio *lock_holder; + struct hash_zone *zone; + bool collides; + + if (list_empty(&lock->duplicate_ring)) + return false; + + lock_holder = list_first_entry(&lock->duplicate_ring, struct data_vio, + hash_lock_entry); + zone = candidate->hash_zone; + collides = !blocks_equal(lock_holder->vio.data, candidate->vio.data); + if (collides) + increment_stat(&zone->statistics.concurrent_hash_collisions); + else + increment_stat(&zone->statistics.concurrent_data_matches); + + return collides; +} + +static inline int assert_hash_lock_preconditions(const struct data_vio *data_vio) +{ + int result; + + /* FIXME: BUG_ON() and/or enter read-only mode? */ + result = VDO_ASSERT(data_vio->hash_lock == NULL, + "must not already hold a hash lock"); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT(list_empty(&data_vio->hash_lock_entry), + "must not already be a member of a hash lock ring"); + if (result != VDO_SUCCESS) + return result; + + return VDO_ASSERT(data_vio->recovery_sequence_number == 0, + "must not hold a recovery lock when getting a hash lock"); +} + +/** + * vdo_acquire_hash_lock() - Acquire or share a lock on a record name. + * @data_vio: The data_vio acquiring a lock on its record name. + * + * Acquire or share a lock on the hash (record name) of the data in a data_vio, updating the + * data_vio to reference the lock. This must only be called in the correct thread for the zone. In + * the unlikely case of a hash collision, this function will succeed, but the data_vio will not get + * a lock reference. + */ +void vdo_acquire_hash_lock(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + struct hash_lock *lock; + int result; + + assert_data_vio_in_hash_zone(data_vio); + + result = assert_hash_lock_preconditions(data_vio); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(data_vio, result); + return; + } + + result = acquire_lock(data_vio->hash_zone, &data_vio->record_name, NULL, &lock); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(data_vio, result); + return; + } + + if (is_hash_collision(lock, data_vio)) { + /* + * Hash collisions are extremely unlikely, but the bogus dedupe would be a data + * corruption. Bypass optimization entirely. We can't compress a data_vio without + * a hash_lock as the compressed write depends on the hash_lock to manage the + * references for the compressed block. + */ + write_data_vio(data_vio); + return; + } + + set_hash_lock(data_vio, lock); + switch (lock->state) { + case VDO_HASH_LOCK_INITIALIZING: + start_querying(lock, data_vio); + return; + + case VDO_HASH_LOCK_QUERYING: + case VDO_HASH_LOCK_WRITING: + case VDO_HASH_LOCK_UPDATING: + case VDO_HASH_LOCK_LOCKING: + case VDO_HASH_LOCK_VERIFYING: + case VDO_HASH_LOCK_UNLOCKING: + /* The lock is busy, and can't be shared yet. */ + wait_on_hash_lock(lock, data_vio); + return; + + case VDO_HASH_LOCK_BYPASSING: + /* We can't use this lock, so bypass optimization entirely. */ + vdo_release_hash_lock(data_vio); + write_data_vio(data_vio); + return; + + case VDO_HASH_LOCK_DEDUPING: + launch_dedupe(lock, data_vio, false); + return; + + default: + /* A lock in this state should not be acquired by new VIOs. */ + report_bogus_lock_state(lock, data_vio); + } +} + +/** + * vdo_release_hash_lock() - Release a data_vio's share of a hash lock, if held, and null out the + * data_vio's reference to it. + * @data_vio: The data_vio releasing its hash lock. + * + * If the data_vio is the only one holding the lock, this also releases any resources or locks used + * by the hash lock (such as a PBN read lock on a block containing data with the same hash) and + * returns the lock to the hash zone's lock pool. + * + * Context: This must only be called in the correct thread for the hash zone. + */ +void vdo_release_hash_lock(struct data_vio *data_vio) +{ + u64 lock_key; + struct hash_lock *lock = data_vio->hash_lock; + struct hash_zone *zone = data_vio->hash_zone; + + if (lock == NULL) + return; + + set_hash_lock(data_vio, NULL); + + if (lock->reference_count > 0) { + /* The lock is still in use by other data_vios. */ + return; + } + + lock_key = hash_lock_key(lock); + if (lock->registered) { + struct hash_lock *removed; + + removed = vdo_int_map_remove(zone->hash_lock_map, lock_key); + VDO_ASSERT_LOG_ONLY(lock == removed, + "hash lock being released must have been mapped"); + } else { + VDO_ASSERT_LOG_ONLY(lock != vdo_int_map_get(zone->hash_lock_map, lock_key), + "unregistered hash lock must not be in the lock map"); + } + + VDO_ASSERT_LOG_ONLY(!vdo_waitq_has_waiters(&lock->waiters), + "hash lock returned to zone must have no waiters"); + VDO_ASSERT_LOG_ONLY((lock->duplicate_lock == NULL), + "hash lock returned to zone must not reference a PBN lock"); + VDO_ASSERT_LOG_ONLY((lock->state == VDO_HASH_LOCK_BYPASSING), + "returned hash lock must not be in use with state %s", + get_hash_lock_state_name(lock->state)); + VDO_ASSERT_LOG_ONLY(list_empty(&lock->pool_node), + "hash lock returned to zone must not be in a pool ring"); + VDO_ASSERT_LOG_ONLY(list_empty(&lock->duplicate_ring), + "hash lock returned to zone must not reference DataVIOs"); + + return_hash_lock_to_pool(zone, lock); +} + +/** + * transfer_allocation_lock() - Transfer a data_vio's downgraded allocation PBN lock to the + * data_vio's hash lock, converting it to a duplicate PBN lock. + * @data_vio: The data_vio holding the allocation lock to transfer. + */ +static void transfer_allocation_lock(struct data_vio *data_vio) +{ + struct allocation *allocation = &data_vio->allocation; + struct hash_lock *hash_lock = data_vio->hash_lock; + + VDO_ASSERT_LOG_ONLY(data_vio->new_mapped.pbn == allocation->pbn, + "transferred lock must be for the block written"); + + allocation->pbn = VDO_ZERO_BLOCK; + + VDO_ASSERT_LOG_ONLY(vdo_is_pbn_read_lock(allocation->lock), + "must have downgraded the allocation lock before transfer"); + + hash_lock->duplicate = data_vio->new_mapped; + data_vio->duplicate = data_vio->new_mapped; + + /* + * Since the lock is being transferred, the holder count doesn't change (and isn't even + * safe to examine on this thread). + */ + hash_lock->duplicate_lock = vdo_forget(allocation->lock); +} + +/** + * vdo_share_compressed_write_lock() - Make a data_vio's hash lock a shared holder of the PBN lock + * on the compressed block to which its data was just written. + * @data_vio: The data_vio which was just compressed. + * @pbn_lock: The PBN lock on the compressed block. + * + * If the lock is still a write lock (as it will be for the first share), it will be converted to a + * read lock. This also reserves a reference count increment for the data_vio. + */ +void vdo_share_compressed_write_lock(struct data_vio *data_vio, + struct pbn_lock *pbn_lock) +{ + bool claimed; + + VDO_ASSERT_LOG_ONLY(vdo_get_duplicate_lock(data_vio) == NULL, + "a duplicate PBN lock should not exist when writing"); + VDO_ASSERT_LOG_ONLY(vdo_is_state_compressed(data_vio->new_mapped.state), + "lock transfer must be for a compressed write"); + assert_data_vio_in_new_mapped_zone(data_vio); + + /* First sharer downgrades the lock. */ + if (!vdo_is_pbn_read_lock(pbn_lock)) + vdo_downgrade_pbn_write_lock(pbn_lock, true); + + /* + * Get a share of the PBN lock, ensuring it cannot be released until after this data_vio + * has had a chance to journal a reference. + */ + data_vio->duplicate = data_vio->new_mapped; + data_vio->hash_lock->duplicate = data_vio->new_mapped; + set_duplicate_lock(data_vio->hash_lock, pbn_lock); + + /* + * Claim a reference for this data_vio. Necessary since another hash_lock might start + * deduplicating against it before our incRef. + */ + claimed = vdo_claim_pbn_lock_increment(pbn_lock); + VDO_ASSERT_LOG_ONLY(claimed, "impossible to fail to claim an initial increment"); +} + +static void start_uds_queue(void *ptr) +{ + /* + * Allow the UDS dedupe worker thread to do memory allocations. It will only do allocations + * during the UDS calls that open or close an index, but those allocations can safely sleep + * while reserving a large amount of memory. We could use an allocations_allowed boolean + * (like the base threads do), but it would be an unnecessary embellishment. + */ + struct vdo_thread *thread = vdo_get_work_queue_owner(vdo_get_current_work_queue()); + + vdo_register_allocating_thread(&thread->allocating_thread, NULL); +} + +static void finish_uds_queue(void *ptr __always_unused) +{ + vdo_unregister_allocating_thread(); +} + +static void close_index(struct hash_zones *zones) + __must_hold(&zones->lock) +{ + int result; + + /* + * Change the index state so that get_index_statistics() will not try to use the index + * session we are closing. + */ + zones->index_state = IS_CHANGING; + /* Close the index session, while not holding the lock. */ + spin_unlock(&zones->lock); + result = uds_close_index(zones->index_session); + + if (result != UDS_SUCCESS) + vdo_log_error_strerror(result, "Error closing index"); + spin_lock(&zones->lock); + zones->index_state = IS_CLOSED; + zones->error_flag |= result != UDS_SUCCESS; + /* ASSERTION: We leave in IS_CLOSED state. */ +} + +static void open_index(struct hash_zones *zones) + __must_hold(&zones->lock) +{ + /* ASSERTION: We enter in IS_CLOSED state. */ + int result; + bool create_flag = zones->create_flag; + + zones->create_flag = false; + /* + * Change the index state so that the it will be reported to the outside world as + * "opening". + */ + zones->index_state = IS_CHANGING; + zones->error_flag = false; + + /* Open the index session, while not holding the lock */ + spin_unlock(&zones->lock); + result = uds_open_index(create_flag ? UDS_CREATE : UDS_LOAD, + &zones->parameters, zones->index_session); + if (result != UDS_SUCCESS) + vdo_log_error_strerror(result, "Error opening index"); + + spin_lock(&zones->lock); + if (!create_flag) { + switch (result) { + case -ENOENT: + /* + * Either there is no index, or there is no way we can recover the index. + * We will be called again and try to create a new index. + */ + zones->index_state = IS_CLOSED; + zones->create_flag = true; + return; + default: + break; + } + } + if (result == UDS_SUCCESS) { + zones->index_state = IS_OPENED; + } else { + zones->index_state = IS_CLOSED; + zones->index_target = IS_CLOSED; + zones->error_flag = true; + spin_unlock(&zones->lock); + vdo_log_info("Setting UDS index target state to error"); + spin_lock(&zones->lock); + } + /* + * ASSERTION: On success, we leave in IS_OPENED state. + * ASSERTION: On failure, we leave in IS_CLOSED state. + */ +} + +static void change_dedupe_state(struct vdo_completion *completion) +{ + struct hash_zones *zones = as_hash_zones(completion); + + spin_lock(&zones->lock); + + /* Loop until the index is in the target state and the create flag is clear. */ + while (vdo_is_state_normal(&zones->state) && + ((zones->index_state != zones->index_target) || zones->create_flag)) { + if (zones->index_state == IS_OPENED) + close_index(zones); + else + open_index(zones); + } + + zones->changing = false; + spin_unlock(&zones->lock); +} + +static void start_expiration_timer(struct dedupe_context *context) +{ + u64 start_time = context->submission_jiffies; + u64 end_time; + + if (!change_timer_state(context->zone, DEDUPE_QUERY_TIMER_IDLE, + DEDUPE_QUERY_TIMER_RUNNING)) + return; + + end_time = max(start_time + vdo_dedupe_index_timeout_jiffies, + jiffies + vdo_dedupe_index_min_timer_jiffies); + mod_timer(&context->zone->timer, end_time); +} + +/** + * report_dedupe_timeouts() - Record and eventually report that some dedupe requests reached their + * expiration time without getting answers, so we timed them out. + * @zones: the hash zones. + * @timeouts: the number of newly timed out requests. + */ +static void report_dedupe_timeouts(struct hash_zones *zones, unsigned int timeouts) +{ + atomic64_add(timeouts, &zones->timeouts); + spin_lock(&zones->lock); + if (__ratelimit(&zones->ratelimiter)) { + u64 unreported = atomic64_read(&zones->timeouts); + + unreported -= zones->reported_timeouts; + vdo_log_debug("UDS index timeout on %llu requests", + (unsigned long long) unreported); + zones->reported_timeouts += unreported; + } + spin_unlock(&zones->lock); +} + +static int initialize_index(struct vdo *vdo, struct hash_zones *zones) +{ + int result; + off_t uds_offset; + struct volume_geometry geometry = vdo->geometry; + static const struct vdo_work_queue_type uds_queue_type = { + .start = start_uds_queue, + .finish = finish_uds_queue, + .max_priority = UDS_Q_MAX_PRIORITY, + .default_priority = UDS_Q_PRIORITY, + }; + + vdo_set_dedupe_index_timeout_interval(vdo_dedupe_index_timeout_interval); + vdo_set_dedupe_index_min_timer_interval(vdo_dedupe_index_min_timer_interval); + + /* + * Since we will save up the timeouts that would have been reported but were ratelimited, + * we don't need to report ratelimiting. + */ + ratelimit_default_init(&zones->ratelimiter); + ratelimit_set_flags(&zones->ratelimiter, RATELIMIT_MSG_ON_RELEASE); + uds_offset = ((vdo_get_index_region_start(geometry) - + geometry.bio_offset) * VDO_BLOCK_SIZE); + zones->parameters = (struct uds_parameters) { + .bdev = vdo->device_config->owned_device->bdev, + .offset = uds_offset, + .size = (vdo_get_index_region_size(geometry) * VDO_BLOCK_SIZE), + .memory_size = geometry.index_config.mem, + .sparse = geometry.index_config.sparse, + .nonce = (u64) geometry.nonce, + }; + + result = uds_create_index_session(&zones->index_session); + if (result != UDS_SUCCESS) + return result; + + result = vdo_make_thread(vdo, vdo->thread_config.dedupe_thread, &uds_queue_type, + 1, NULL); + if (result != VDO_SUCCESS) { + uds_destroy_index_session(vdo_forget(zones->index_session)); + vdo_log_error("UDS index queue initialization failed (%d)", result); + return result; + } + + vdo_initialize_completion(&zones->completion, vdo, VDO_HASH_ZONES_COMPLETION); + vdo_set_completion_callback(&zones->completion, change_dedupe_state, + vdo->thread_config.dedupe_thread); + return VDO_SUCCESS; +} + +/** + * finish_index_operation() - This is the UDS callback for index queries. + * @request: The uds request which has just completed. + */ +static void finish_index_operation(struct uds_request *request) +{ + struct dedupe_context *context = container_of(request, struct dedupe_context, + request); + + if (change_context_state(context, DEDUPE_CONTEXT_PENDING, + DEDUPE_CONTEXT_COMPLETE)) { + /* + * This query has not timed out, so send its data_vio back to its hash zone to + * process the results. + */ + continue_data_vio(context->requestor); + return; + } + + /* + * This query has timed out, so try to mark it complete and hence eligible for reuse. Its + * data_vio has already moved on. + */ + if (!change_context_state(context, DEDUPE_CONTEXT_TIMED_OUT, + DEDUPE_CONTEXT_TIMED_OUT_COMPLETE)) { + VDO_ASSERT_LOG_ONLY(false, "uds request was timed out (state %d)", + atomic_read(&context->state)); + } + + vdo_funnel_queue_put(context->zone->timed_out_complete, &context->queue_entry); +} + +/** + * check_for_drain_complete() - Check whether this zone has drained. + * @zone: The zone to check. + */ +static void check_for_drain_complete(struct hash_zone *zone) +{ + data_vio_count_t recycled = 0; + + if (!vdo_is_state_draining(&zone->state)) + return; + + if ((atomic_read(&zone->timer_state) == DEDUPE_QUERY_TIMER_IDLE) || + change_timer_state(zone, DEDUPE_QUERY_TIMER_RUNNING, + DEDUPE_QUERY_TIMER_IDLE)) { + del_timer_sync(&zone->timer); + } else { + /* + * There is an in flight time-out, which must get processed before we can continue. + */ + return; + } + + for (;;) { + struct dedupe_context *context; + struct funnel_queue_entry *entry; + + entry = vdo_funnel_queue_poll(zone->timed_out_complete); + if (entry == NULL) + break; + + context = container_of(entry, struct dedupe_context, queue_entry); + atomic_set(&context->state, DEDUPE_CONTEXT_IDLE); + list_add(&context->list_entry, &zone->available); + recycled++; + } + + if (recycled > 0) + WRITE_ONCE(zone->active, zone->active - recycled); + VDO_ASSERT_LOG_ONLY(READ_ONCE(zone->active) == 0, "all contexts inactive"); + vdo_finish_draining(&zone->state); +} + +static void timeout_index_operations_callback(struct vdo_completion *completion) +{ + struct dedupe_context *context, *tmp; + struct hash_zone *zone = as_hash_zone(completion); + u64 timeout_jiffies = msecs_to_jiffies(vdo_dedupe_index_timeout_interval); + unsigned long cutoff = jiffies - timeout_jiffies; + unsigned int timed_out = 0; + + atomic_set(&zone->timer_state, DEDUPE_QUERY_TIMER_IDLE); + list_for_each_entry_safe(context, tmp, &zone->pending, list_entry) { + if (cutoff <= context->submission_jiffies) { + /* + * We have reached the oldest query which has not timed out yet, so restart + * the timer. + */ + start_expiration_timer(context); + break; + } + + if (!change_context_state(context, DEDUPE_CONTEXT_PENDING, + DEDUPE_CONTEXT_TIMED_OUT)) { + /* + * This context completed between the time the timeout fired, and now. We + * can treat it as a successful query, its requestor is already enqueued + * to process it. + */ + continue; + } + + /* + * Remove this context from the pending list so we won't look at it again on a + * subsequent timeout. Once the index completes it, it will be reused. Meanwhile, + * send its requestor on its way. + */ + list_del_init(&context->list_entry); + continue_data_vio(context->requestor); + timed_out++; + } + + if (timed_out > 0) + report_dedupe_timeouts(completion->vdo->hash_zones, timed_out); + + check_for_drain_complete(zone); +} + +static void timeout_index_operations(struct timer_list *t) +{ + struct hash_zone *zone = from_timer(zone, t, timer); + + if (change_timer_state(zone, DEDUPE_QUERY_TIMER_RUNNING, + DEDUPE_QUERY_TIMER_FIRED)) + vdo_launch_completion(&zone->completion); +} + +static int __must_check initialize_zone(struct vdo *vdo, struct hash_zones *zones, + zone_count_t zone_number) +{ + int result; + data_vio_count_t i; + struct hash_zone *zone = &zones->zones[zone_number]; + + result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, &zone->hash_lock_map); + if (result != VDO_SUCCESS) + return result; + + vdo_set_admin_state_code(&zone->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + zone->zone_number = zone_number; + zone->thread_id = vdo->thread_config.hash_zone_threads[zone_number]; + vdo_initialize_completion(&zone->completion, vdo, VDO_HASH_ZONE_COMPLETION); + vdo_set_completion_callback(&zone->completion, timeout_index_operations_callback, + zone->thread_id); + INIT_LIST_HEAD(&zone->lock_pool); + result = vdo_allocate(LOCK_POOL_CAPACITY, struct hash_lock, "hash_lock array", + &zone->lock_array); + if (result != VDO_SUCCESS) + return result; + + for (i = 0; i < LOCK_POOL_CAPACITY; i++) + return_hash_lock_to_pool(zone, &zone->lock_array[i]); + + INIT_LIST_HEAD(&zone->available); + INIT_LIST_HEAD(&zone->pending); + result = vdo_make_funnel_queue(&zone->timed_out_complete); + if (result != VDO_SUCCESS) + return result; + + timer_setup(&zone->timer, timeout_index_operations, 0); + + for (i = 0; i < MAXIMUM_VDO_USER_VIOS; i++) { + struct dedupe_context *context = &zone->contexts[i]; + + context->zone = zone; + context->request.callback = finish_index_operation; + context->request.session = zones->index_session; + list_add(&context->list_entry, &zone->available); + } + + return vdo_make_default_thread(vdo, zone->thread_id); +} + +/** get_thread_id_for_zone() - Implements vdo_zone_thread_getter_fn. */ +static thread_id_t get_thread_id_for_zone(void *context, zone_count_t zone_number) +{ + struct hash_zones *zones = context; + + return zones->zones[zone_number].thread_id; +} + +/** + * vdo_make_hash_zones() - Create the hash zones. + * + * @vdo: The vdo to which the zone will belong. + * @zones_ptr: A pointer to hold the zones. + * + * Return: VDO_SUCCESS or an error code. + */ +int vdo_make_hash_zones(struct vdo *vdo, struct hash_zones **zones_ptr) +{ + int result; + struct hash_zones *zones; + zone_count_t z; + zone_count_t zone_count = vdo->thread_config.hash_zone_count; + + if (zone_count == 0) + return VDO_SUCCESS; + + result = vdo_allocate_extended(struct hash_zones, zone_count, struct hash_zone, + __func__, &zones); + if (result != VDO_SUCCESS) + return result; + + result = initialize_index(vdo, zones); + if (result != VDO_SUCCESS) { + vdo_free(zones); + return result; + } + + vdo_set_admin_state_code(&zones->state, VDO_ADMIN_STATE_NEW); + + zones->zone_count = zone_count; + for (z = 0; z < zone_count; z++) { + result = initialize_zone(vdo, zones, z); + if (result != VDO_SUCCESS) { + vdo_free_hash_zones(zones); + return result; + } + } + + result = vdo_make_action_manager(zones->zone_count, get_thread_id_for_zone, + vdo->thread_config.admin_thread, zones, NULL, + vdo, &zones->manager); + if (result != VDO_SUCCESS) { + vdo_free_hash_zones(zones); + return result; + } + + *zones_ptr = zones; + return VDO_SUCCESS; +} + +void vdo_finish_dedupe_index(struct hash_zones *zones) +{ + if (zones == NULL) + return; + + uds_destroy_index_session(vdo_forget(zones->index_session)); +} + +/** + * vdo_free_hash_zones() - Free the hash zones. + * @zones: The zone to free. + */ +void vdo_free_hash_zones(struct hash_zones *zones) +{ + zone_count_t i; + + if (zones == NULL) + return; + + vdo_free(vdo_forget(zones->manager)); + + for (i = 0; i < zones->zone_count; i++) { + struct hash_zone *zone = &zones->zones[i]; + + vdo_free_funnel_queue(vdo_forget(zone->timed_out_complete)); + vdo_int_map_free(vdo_forget(zone->hash_lock_map)); + vdo_free(vdo_forget(zone->lock_array)); + } + + if (zones->index_session != NULL) + vdo_finish_dedupe_index(zones); + + ratelimit_state_exit(&zones->ratelimiter); + vdo_free(zones); +} + +static void initiate_suspend_index(struct admin_state *state) +{ + struct hash_zones *zones = container_of(state, struct hash_zones, state); + enum index_state index_state; + + spin_lock(&zones->lock); + index_state = zones->index_state; + spin_unlock(&zones->lock); + + if (index_state != IS_CLOSED) { + bool save = vdo_is_state_saving(&zones->state); + int result; + + result = uds_suspend_index_session(zones->index_session, save); + if (result != UDS_SUCCESS) + vdo_log_error_strerror(result, "Error suspending dedupe index"); + } + + vdo_finish_draining(state); +} + +/** + * suspend_index() - Suspend the UDS index prior to draining hash zones. + * + * Implements vdo_action_preamble_fn + */ +static void suspend_index(void *context, struct vdo_completion *completion) +{ + struct hash_zones *zones = context; + + vdo_start_draining(&zones->state, + vdo_get_current_manager_operation(zones->manager), completion, + initiate_suspend_index); +} + +/** + * initiate_drain() - Initiate a drain. + * + * Implements vdo_admin_initiator_fn. + */ +static void initiate_drain(struct admin_state *state) +{ + check_for_drain_complete(container_of(state, struct hash_zone, state)); +} + +/** + * drain_hash_zone() - Drain a hash zone. + * + * Implements vdo_zone_action_fn. + */ +static void drain_hash_zone(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct hash_zones *zones = context; + + vdo_start_draining(&zones->zones[zone_number].state, + vdo_get_current_manager_operation(zones->manager), parent, + initiate_drain); +} + +/** vdo_drain_hash_zones() - Drain all hash zones. */ +void vdo_drain_hash_zones(struct hash_zones *zones, struct vdo_completion *parent) +{ + vdo_schedule_operation(zones->manager, parent->vdo->suspend_type, suspend_index, + drain_hash_zone, NULL, parent); +} + +static void launch_dedupe_state_change(struct hash_zones *zones) + __must_hold(&zones->lock) +{ + /* ASSERTION: We enter with the lock held. */ + if (zones->changing || !vdo_is_state_normal(&zones->state)) + /* Either a change is already in progress, or changes are not allowed. */ + return; + + if (zones->create_flag || (zones->index_state != zones->index_target)) { + zones->changing = true; + vdo_launch_completion(&zones->completion); + return; + } + + /* ASSERTION: We exit with the lock held. */ +} + +/** + * resume_index() - Resume the UDS index prior to resuming hash zones. + * + * Implements vdo_action_preamble_fn + */ +static void resume_index(void *context, struct vdo_completion *parent) +{ + struct hash_zones *zones = context; + struct device_config *config = parent->vdo->device_config; + int result; + + zones->parameters.bdev = config->owned_device->bdev; + result = uds_resume_index_session(zones->index_session, zones->parameters.bdev); + if (result != UDS_SUCCESS) + vdo_log_error_strerror(result, "Error resuming dedupe index"); + + spin_lock(&zones->lock); + vdo_resume_if_quiescent(&zones->state); + + if (config->deduplication) { + zones->index_target = IS_OPENED; + WRITE_ONCE(zones->dedupe_flag, true); + } else { + zones->index_target = IS_CLOSED; + } + + launch_dedupe_state_change(zones); + spin_unlock(&zones->lock); + + vdo_finish_completion(parent); +} + +/** + * resume_hash_zone() - Resume a hash zone. + * + * Implements vdo_zone_action_fn. + */ +static void resume_hash_zone(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct hash_zone *zone = &(((struct hash_zones *) context)->zones[zone_number]); + + vdo_fail_completion(parent, vdo_resume_if_quiescent(&zone->state)); +} + +/** + * vdo_resume_hash_zones() - Resume a set of hash zones. + * @zones: The hash zones to resume. + * @parent: The object to notify when the zones have resumed. + */ +void vdo_resume_hash_zones(struct hash_zones *zones, struct vdo_completion *parent) +{ + if (vdo_is_read_only(parent->vdo)) { + vdo_launch_completion(parent); + return; + } + + vdo_schedule_operation(zones->manager, VDO_ADMIN_STATE_RESUMING, resume_index, + resume_hash_zone, NULL, parent); +} + +/** + * get_hash_zone_statistics() - Add the statistics for this hash zone to the tally for all zones. + * @zone: The hash zone to query. + * @tally: The tally + */ +static void get_hash_zone_statistics(const struct hash_zone *zone, + struct hash_lock_statistics *tally) +{ + const struct hash_lock_statistics *stats = &zone->statistics; + + tally->dedupe_advice_valid += READ_ONCE(stats->dedupe_advice_valid); + tally->dedupe_advice_stale += READ_ONCE(stats->dedupe_advice_stale); + tally->concurrent_data_matches += READ_ONCE(stats->concurrent_data_matches); + tally->concurrent_hash_collisions += READ_ONCE(stats->concurrent_hash_collisions); + tally->curr_dedupe_queries += READ_ONCE(zone->active); +} + +static void get_index_statistics(struct hash_zones *zones, + struct index_statistics *stats) +{ + enum index_state state; + struct uds_index_stats index_stats; + int result; + + spin_lock(&zones->lock); + state = zones->index_state; + spin_unlock(&zones->lock); + + if (state != IS_OPENED) + return; + + result = uds_get_index_session_stats(zones->index_session, &index_stats); + if (result != UDS_SUCCESS) { + vdo_log_error_strerror(result, "Error reading index stats"); + return; + } + + stats->entries_indexed = index_stats.entries_indexed; + stats->posts_found = index_stats.posts_found; + stats->posts_not_found = index_stats.posts_not_found; + stats->queries_found = index_stats.queries_found; + stats->queries_not_found = index_stats.queries_not_found; + stats->updates_found = index_stats.updates_found; + stats->updates_not_found = index_stats.updates_not_found; + stats->entries_discarded = index_stats.entries_discarded; +} + +/** + * vdo_get_dedupe_statistics() - Tally the statistics from all the hash zones and the UDS index. + * @hash_zones: The hash zones to query + * + * Return: The sum of the hash lock statistics from all hash zones plus the statistics from the UDS + * index + */ +void vdo_get_dedupe_statistics(struct hash_zones *zones, struct vdo_statistics *stats) + +{ + zone_count_t zone; + + for (zone = 0; zone < zones->zone_count; zone++) + get_hash_zone_statistics(&zones->zones[zone], &stats->hash_lock); + + get_index_statistics(zones, &stats->index); + + /* + * zones->timeouts gives the number of timeouts, and dedupe_context_busy gives the number + * of queries not made because of earlier timeouts. + */ + stats->dedupe_advice_timeouts = + (atomic64_read(&zones->timeouts) + atomic64_read(&zones->dedupe_context_busy)); +} + +/** + * vdo_select_hash_zone() - Select the hash zone responsible for locking a given record name. + * @zones: The hash_zones from which to select. + * @name: The record name. + * + * Return: The hash zone responsible for the record name. + */ +struct hash_zone *vdo_select_hash_zone(struct hash_zones *zones, + const struct uds_record_name *name) +{ + /* + * Use a fragment of the record name as a hash code. Eight bits of hash should suffice + * since the number of hash zones is small. + * TODO: Verify that the first byte is independent enough. + */ + u32 hash = name->name[0]; + + /* + * Scale the 8-bit hash fragment to a zone index by treating it as a binary fraction and + * multiplying that by the zone count. If the hash is uniformly distributed over [0 .. + * 2^8-1], then (hash * count / 2^8) should be uniformly distributed over [0 .. count-1]. + * The multiply and shift is much faster than a divide (modulus) on X86 CPUs. + */ + hash = (hash * zones->zone_count) >> 8; + return &zones->zones[hash]; +} + +/** + * dump_hash_lock() - Dump a compact description of hash_lock to the log if the lock is not on the + * free list. + * @lock: The hash lock to dump. + */ +static void dump_hash_lock(const struct hash_lock *lock) +{ + const char *state; + + if (!list_empty(&lock->pool_node)) { + /* This lock is on the free list. */ + return; + } + + /* + * Necessarily cryptic since we can log a lot of these. First three chars of state is + * unambiguous. 'U' indicates a lock not registered in the map. + */ + state = get_hash_lock_state_name(lock->state); + vdo_log_info(" hl %px: %3.3s %c%llu/%u rc=%u wc=%zu agt=%px", + lock, state, (lock->registered ? 'D' : 'U'), + (unsigned long long) lock->duplicate.pbn, + lock->duplicate.state, lock->reference_count, + vdo_waitq_num_waiters(&lock->waiters), lock->agent); +} + +static const char *index_state_to_string(struct hash_zones *zones, + enum index_state state) +{ + if (!vdo_is_state_normal(&zones->state)) + return SUSPENDED; + + switch (state) { + case IS_CLOSED: + return zones->error_flag ? ERROR : CLOSED; + case IS_CHANGING: + return zones->index_target == IS_OPENED ? OPENING : CLOSING; + case IS_OPENED: + return READ_ONCE(zones->dedupe_flag) ? ONLINE : OFFLINE; + default: + return UNKNOWN; + } +} + +/** + * dump_hash_zone() - Dump information about a hash zone to the log for debugging. + * @zone: The zone to dump. + */ +static void dump_hash_zone(const struct hash_zone *zone) +{ + data_vio_count_t i; + + if (zone->hash_lock_map == NULL) { + vdo_log_info("struct hash_zone %u: NULL map", zone->zone_number); + return; + } + + vdo_log_info("struct hash_zone %u: mapSize=%zu", + zone->zone_number, vdo_int_map_size(zone->hash_lock_map)); + for (i = 0; i < LOCK_POOL_CAPACITY; i++) + dump_hash_lock(&zone->lock_array[i]); +} + +/** + * vdo_dump_hash_zones() - Dump information about the hash zones to the log for debugging. + * @zones: The zones to dump. + */ +void vdo_dump_hash_zones(struct hash_zones *zones) +{ + const char *state, *target; + zone_count_t zone; + + spin_lock(&zones->lock); + state = index_state_to_string(zones, zones->index_state); + target = (zones->changing ? index_state_to_string(zones, zones->index_target) : NULL); + spin_unlock(&zones->lock); + + vdo_log_info("UDS index: state: %s", state); + if (target != NULL) + vdo_log_info("UDS index: changing to state: %s", target); + + for (zone = 0; zone < zones->zone_count; zone++) + dump_hash_zone(&zones->zones[zone]); +} + +void vdo_set_dedupe_index_timeout_interval(unsigned int value) +{ + u64 alb_jiffies; + + /* Arbitrary maximum value is two minutes */ + if (value > 120000) + value = 120000; + /* Arbitrary minimum value is 2 jiffies */ + alb_jiffies = msecs_to_jiffies(value); + + if (alb_jiffies < 2) { + alb_jiffies = 2; + value = jiffies_to_msecs(alb_jiffies); + } + vdo_dedupe_index_timeout_interval = value; + vdo_dedupe_index_timeout_jiffies = alb_jiffies; +} + +void vdo_set_dedupe_index_min_timer_interval(unsigned int value) +{ + u64 min_jiffies; + + /* Arbitrary maximum value is one second */ + if (value > 1000) + value = 1000; + + /* Arbitrary minimum value is 2 jiffies */ + min_jiffies = msecs_to_jiffies(value); + + if (min_jiffies < 2) { + min_jiffies = 2; + value = jiffies_to_msecs(min_jiffies); + } + + vdo_dedupe_index_min_timer_interval = value; + vdo_dedupe_index_min_timer_jiffies = min_jiffies; +} + +/** + * acquire_context() - Acquire a dedupe context from a hash_zone if any are available. + * @zone: the hash zone + * + * Return: A dedupe_context or NULL if none are available + */ +static struct dedupe_context * __must_check acquire_context(struct hash_zone *zone) +{ + struct dedupe_context *context; + struct funnel_queue_entry *entry; + + assert_in_hash_zone(zone, __func__); + + if (!list_empty(&zone->available)) { + WRITE_ONCE(zone->active, zone->active + 1); + context = list_first_entry(&zone->available, struct dedupe_context, + list_entry); + list_del_init(&context->list_entry); + return context; + } + + entry = vdo_funnel_queue_poll(zone->timed_out_complete); + return ((entry == NULL) ? + NULL : container_of(entry, struct dedupe_context, queue_entry)); +} + +static void prepare_uds_request(struct uds_request *request, struct data_vio *data_vio, + enum uds_request_type operation) +{ + request->record_name = data_vio->record_name; + request->type = operation; + if ((operation == UDS_POST) || (operation == UDS_UPDATE)) { + size_t offset = 0; + struct uds_record_data *encoding = &request->new_metadata; + + encoding->data[offset++] = UDS_ADVICE_VERSION; + encoding->data[offset++] = data_vio->new_mapped.state; + put_unaligned_le64(data_vio->new_mapped.pbn, &encoding->data[offset]); + offset += sizeof(u64); + BUG_ON(offset != UDS_ADVICE_SIZE); + } +} + +/* + * The index operation will inquire about data_vio.record_name, providing (if the operation is + * appropriate) advice from the data_vio's new_mapped fields. The advice found in the index (or + * NULL if none) will be returned via receive_data_vio_dedupe_advice(). dedupe_context.status is + * set to the return status code of any asynchronous index processing. + */ +static void query_index(struct data_vio *data_vio, enum uds_request_type operation) +{ + int result; + struct dedupe_context *context; + struct vdo *vdo = vdo_from_data_vio(data_vio); + struct hash_zone *zone = data_vio->hash_zone; + + assert_data_vio_in_hash_zone(data_vio); + + if (!READ_ONCE(vdo->hash_zones->dedupe_flag)) { + continue_data_vio(data_vio); + return; + } + + context = acquire_context(zone); + if (context == NULL) { + atomic64_inc(&vdo->hash_zones->dedupe_context_busy); + continue_data_vio(data_vio); + return; + } + + data_vio->dedupe_context = context; + context->requestor = data_vio; + context->submission_jiffies = jiffies; + prepare_uds_request(&context->request, data_vio, operation); + atomic_set(&context->state, DEDUPE_CONTEXT_PENDING); + list_add_tail(&context->list_entry, &zone->pending); + start_expiration_timer(context); + result = uds_launch_request(&context->request); + if (result != UDS_SUCCESS) { + context->request.status = result; + finish_index_operation(&context->request); + } +} + +static void set_target_state(struct hash_zones *zones, enum index_state target, + bool change_dedupe, bool dedupe, bool set_create) +{ + const char *old_state, *new_state; + + spin_lock(&zones->lock); + old_state = index_state_to_string(zones, zones->index_target); + if (change_dedupe) + WRITE_ONCE(zones->dedupe_flag, dedupe); + + if (set_create) + zones->create_flag = true; + + zones->index_target = target; + launch_dedupe_state_change(zones); + new_state = index_state_to_string(zones, zones->index_target); + spin_unlock(&zones->lock); + + if (old_state != new_state) + vdo_log_info("Setting UDS index target state to %s", new_state); +} + +const char *vdo_get_dedupe_index_state_name(struct hash_zones *zones) +{ + const char *state; + + spin_lock(&zones->lock); + state = index_state_to_string(zones, zones->index_state); + spin_unlock(&zones->lock); + + return state; +} + +/* Handle a dmsetup message relevant to the index. */ +int vdo_message_dedupe_index(struct hash_zones *zones, const char *name) +{ + if (strcasecmp(name, "index-close") == 0) { + set_target_state(zones, IS_CLOSED, false, false, false); + return 0; + } else if (strcasecmp(name, "index-create") == 0) { + set_target_state(zones, IS_OPENED, false, false, true); + return 0; + } else if (strcasecmp(name, "index-disable") == 0) { + set_target_state(zones, IS_OPENED, true, false, false); + return 0; + } else if (strcasecmp(name, "index-enable") == 0) { + set_target_state(zones, IS_OPENED, true, true, false); + return 0; + } + + return -EINVAL; +} + +void vdo_set_dedupe_state_normal(struct hash_zones *zones) +{ + vdo_set_admin_state_code(&zones->state, VDO_ADMIN_STATE_NORMAL_OPERATION); +} + +/* If create_flag, create a new index without first attempting to load an existing index. */ +void vdo_start_dedupe_index(struct hash_zones *zones, bool create_flag) +{ + set_target_state(zones, IS_OPENED, true, true, create_flag); +} diff --git a/drivers/md/dm-vdo/dedupe.h b/drivers/md/dm-vdo/dedupe.h new file mode 100644 index 000000000000..9000d6f3eece --- /dev/null +++ b/drivers/md/dm-vdo/dedupe.h @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_DEDUPE_H +#define VDO_DEDUPE_H + +#include +#include + +#include "indexer.h" + +#include "admin-state.h" +#include "constants.h" +#include "statistics.h" +#include "types.h" +#include "wait-queue.h" + +struct dedupe_context { + struct hash_zone *zone; + struct uds_request request; + struct list_head list_entry; + struct funnel_queue_entry queue_entry; + u64 submission_jiffies; + struct data_vio *requestor; + atomic_t state; +}; + +struct hash_lock; + +struct hash_zone { + /* Which hash zone this is */ + zone_count_t zone_number; + + /* The administrative state of the zone */ + struct admin_state state; + + /* The thread ID for this zone */ + thread_id_t thread_id; + + /* Mapping from record name fields to hash_locks */ + struct int_map *hash_lock_map; + + /* List containing all unused hash_locks */ + struct list_head lock_pool; + + /* + * Statistics shared by all hash locks in this zone. Only modified on the hash zone thread, + * but queried by other threads. + */ + struct hash_lock_statistics statistics; + + /* Array of all hash_locks */ + struct hash_lock *lock_array; + + /* These fields are used to manage the dedupe contexts */ + struct list_head available; + struct list_head pending; + struct funnel_queue *timed_out_complete; + struct timer_list timer; + struct vdo_completion completion; + unsigned int active; + atomic_t timer_state; + + /* The dedupe contexts for querying the index from this zone */ + struct dedupe_context contexts[MAXIMUM_VDO_USER_VIOS]; +}; + +struct hash_zones; + +struct pbn_lock * __must_check vdo_get_duplicate_lock(struct data_vio *data_vio); + +void vdo_acquire_hash_lock(struct vdo_completion *completion); +void vdo_continue_hash_lock(struct vdo_completion *completion); +void vdo_release_hash_lock(struct data_vio *data_vio); +void vdo_clean_failed_hash_lock(struct data_vio *data_vio); +void vdo_share_compressed_write_lock(struct data_vio *data_vio, + struct pbn_lock *pbn_lock); + +int __must_check vdo_make_hash_zones(struct vdo *vdo, struct hash_zones **zones_ptr); + +void vdo_free_hash_zones(struct hash_zones *zones); + +void vdo_drain_hash_zones(struct hash_zones *zones, struct vdo_completion *parent); + +void vdo_get_dedupe_statistics(struct hash_zones *zones, struct vdo_statistics *stats); + +struct hash_zone * __must_check vdo_select_hash_zone(struct hash_zones *zones, + const struct uds_record_name *name); + +void vdo_dump_hash_zones(struct hash_zones *zones); + +const char *vdo_get_dedupe_index_state_name(struct hash_zones *zones); + +u64 vdo_get_dedupe_index_timeout_count(struct hash_zones *zones); + +int vdo_message_dedupe_index(struct hash_zones *zones, const char *name); + +void vdo_set_dedupe_state_normal(struct hash_zones *zones); + +void vdo_start_dedupe_index(struct hash_zones *zones, bool create_flag); + +void vdo_resume_hash_zones(struct hash_zones *zones, struct vdo_completion *parent); + +void vdo_finish_dedupe_index(struct hash_zones *zones); + +/* Interval (in milliseconds) from submission until switching to fast path and skipping UDS. */ +extern unsigned int vdo_dedupe_index_timeout_interval; + +/* + * Minimum time interval (in milliseconds) between timer invocations to check for requests waiting + * for UDS that should now time out. + */ +extern unsigned int vdo_dedupe_index_min_timer_interval; + +void vdo_set_dedupe_index_timeout_interval(unsigned int value); +void vdo_set_dedupe_index_min_timer_interval(unsigned int value); + +#endif /* VDO_DEDUPE_H */ diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c new file mode 100644 index 000000000000..5a4b0a927f56 --- /dev/null +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -0,0 +1,2910 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "admin-state.h" +#include "block-map.h" +#include "completion.h" +#include "constants.h" +#include "data-vio.h" +#include "dedupe.h" +#include "dump.h" +#include "encodings.h" +#include "errors.h" +#include "flush.h" +#include "io-submitter.h" +#include "logger.h" +#include "memory-alloc.h" +#include "message-stats.h" +#include "recovery-journal.h" +#include "repair.h" +#include "slab-depot.h" +#include "status-codes.h" +#include "string-utils.h" +#include "thread-device.h" +#include "thread-registry.h" +#include "thread-utils.h" +#include "types.h" +#include "vdo.h" +#include "vio.h" + +enum admin_phases { + GROW_LOGICAL_PHASE_START, + GROW_LOGICAL_PHASE_GROW_BLOCK_MAP, + GROW_LOGICAL_PHASE_END, + GROW_LOGICAL_PHASE_ERROR, + GROW_PHYSICAL_PHASE_START, + GROW_PHYSICAL_PHASE_COPY_SUMMARY, + GROW_PHYSICAL_PHASE_UPDATE_COMPONENTS, + GROW_PHYSICAL_PHASE_USE_NEW_SLABS, + GROW_PHYSICAL_PHASE_END, + GROW_PHYSICAL_PHASE_ERROR, + LOAD_PHASE_START, + LOAD_PHASE_LOAD_DEPOT, + LOAD_PHASE_MAKE_DIRTY, + LOAD_PHASE_PREPARE_TO_ALLOCATE, + LOAD_PHASE_SCRUB_SLABS, + LOAD_PHASE_DATA_REDUCTION, + LOAD_PHASE_FINISHED, + LOAD_PHASE_DRAIN_JOURNAL, + LOAD_PHASE_WAIT_FOR_READ_ONLY, + PRE_LOAD_PHASE_START, + PRE_LOAD_PHASE_LOAD_COMPONENTS, + PRE_LOAD_PHASE_END, + PREPARE_GROW_PHYSICAL_PHASE_START, + RESUME_PHASE_START, + RESUME_PHASE_ALLOW_READ_ONLY_MODE, + RESUME_PHASE_DEDUPE, + RESUME_PHASE_DEPOT, + RESUME_PHASE_JOURNAL, + RESUME_PHASE_BLOCK_MAP, + RESUME_PHASE_LOGICAL_ZONES, + RESUME_PHASE_PACKER, + RESUME_PHASE_FLUSHER, + RESUME_PHASE_DATA_VIOS, + RESUME_PHASE_END, + SUSPEND_PHASE_START, + SUSPEND_PHASE_PACKER, + SUSPEND_PHASE_DATA_VIOS, + SUSPEND_PHASE_DEDUPE, + SUSPEND_PHASE_FLUSHES, + SUSPEND_PHASE_LOGICAL_ZONES, + SUSPEND_PHASE_BLOCK_MAP, + SUSPEND_PHASE_JOURNAL, + SUSPEND_PHASE_DEPOT, + SUSPEND_PHASE_READ_ONLY_WAIT, + SUSPEND_PHASE_WRITE_SUPER_BLOCK, + SUSPEND_PHASE_END, +}; + +static const char * const ADMIN_PHASE_NAMES[] = { + "GROW_LOGICAL_PHASE_START", + "GROW_LOGICAL_PHASE_GROW_BLOCK_MAP", + "GROW_LOGICAL_PHASE_END", + "GROW_LOGICAL_PHASE_ERROR", + "GROW_PHYSICAL_PHASE_START", + "GROW_PHYSICAL_PHASE_COPY_SUMMARY", + "GROW_PHYSICAL_PHASE_UPDATE_COMPONENTS", + "GROW_PHYSICAL_PHASE_USE_NEW_SLABS", + "GROW_PHYSICAL_PHASE_END", + "GROW_PHYSICAL_PHASE_ERROR", + "LOAD_PHASE_START", + "LOAD_PHASE_LOAD_DEPOT", + "LOAD_PHASE_MAKE_DIRTY", + "LOAD_PHASE_PREPARE_TO_ALLOCATE", + "LOAD_PHASE_SCRUB_SLABS", + "LOAD_PHASE_DATA_REDUCTION", + "LOAD_PHASE_FINISHED", + "LOAD_PHASE_DRAIN_JOURNAL", + "LOAD_PHASE_WAIT_FOR_READ_ONLY", + "PRE_LOAD_PHASE_START", + "PRE_LOAD_PHASE_LOAD_COMPONENTS", + "PRE_LOAD_PHASE_END", + "PREPARE_GROW_PHYSICAL_PHASE_START", + "RESUME_PHASE_START", + "RESUME_PHASE_ALLOW_READ_ONLY_MODE", + "RESUME_PHASE_DEDUPE", + "RESUME_PHASE_DEPOT", + "RESUME_PHASE_JOURNAL", + "RESUME_PHASE_BLOCK_MAP", + "RESUME_PHASE_LOGICAL_ZONES", + "RESUME_PHASE_PACKER", + "RESUME_PHASE_FLUSHER", + "RESUME_PHASE_DATA_VIOS", + "RESUME_PHASE_END", + "SUSPEND_PHASE_START", + "SUSPEND_PHASE_PACKER", + "SUSPEND_PHASE_DATA_VIOS", + "SUSPEND_PHASE_DEDUPE", + "SUSPEND_PHASE_FLUSHES", + "SUSPEND_PHASE_LOGICAL_ZONES", + "SUSPEND_PHASE_BLOCK_MAP", + "SUSPEND_PHASE_JOURNAL", + "SUSPEND_PHASE_DEPOT", + "SUSPEND_PHASE_READ_ONLY_WAIT", + "SUSPEND_PHASE_WRITE_SUPER_BLOCK", + "SUSPEND_PHASE_END", +}; + +/* If we bump this, update the arrays below */ +#define TABLE_VERSION 4 + +/* arrays for handling different table versions */ +static const u8 REQUIRED_ARGC[] = { 10, 12, 9, 7, 6 }; +/* pool name no longer used. only here for verification of older versions */ +static const u8 POOL_NAME_ARG_INDEX[] = { 8, 10, 8 }; + +/* + * Track in-use instance numbers using a flat bit array. + * + * O(n) run time isn't ideal, but if we have 1000 VDO devices in use simultaneously we still only + * need to scan 16 words, so it's not likely to be a big deal compared to other resource usage. + */ + +/* + * This minimum size for the bit array creates a numbering space of 0-999, which allows + * successive starts of the same volume to have different instance numbers in any + * reasonably-sized test. Changing instances on restart allows vdoMonReport to detect that + * the ephemeral stats have reset to zero. + */ +#define BIT_COUNT_MINIMUM 1000 +/* Grow the bit array by this many bits when needed */ +#define BIT_COUNT_INCREMENT 100 + +struct instance_tracker { + unsigned int bit_count; + unsigned long *words; + unsigned int count; + unsigned int next; +}; + +static DEFINE_MUTEX(instances_lock); +static struct instance_tracker instances; + +/** + * free_device_config() - Free a device config created by parse_device_config(). + * @config: The config to free. + */ +static void free_device_config(struct device_config *config) +{ + if (config == NULL) + return; + + if (config->owned_device != NULL) + dm_put_device(config->owning_target, config->owned_device); + + vdo_free(config->parent_device_name); + vdo_free(config->original_string); + + /* Reduce the chance a use-after-free (as in BZ 1669960) happens to work. */ + memset(config, 0, sizeof(*config)); + vdo_free(config); +} + +/** + * get_version_number() - Decide the version number from argv. + * + * @argc: The number of table values. + * @argv: The array of table values. + * @error_ptr: A pointer to return a error string in. + * @version_ptr: A pointer to return the version. + * + * Return: VDO_SUCCESS or an error code. + */ +static int get_version_number(int argc, char **argv, char **error_ptr, + unsigned int *version_ptr) +{ + /* version, if it exists, is in a form of V */ + if (sscanf(argv[0], "V%u", version_ptr) == 1) { + if (*version_ptr < 1 || *version_ptr > TABLE_VERSION) { + *error_ptr = "Unknown version number detected"; + return VDO_BAD_CONFIGURATION; + } + } else { + /* V0 actually has no version number in the table string */ + *version_ptr = 0; + } + + /* + * V0 and V1 have no optional parameters. There will always be a parameter for thread + * config, even if it's a "." to show it's an empty list. + */ + if (*version_ptr <= 1) { + if (argc != REQUIRED_ARGC[*version_ptr]) { + *error_ptr = "Incorrect number of arguments for version"; + return VDO_BAD_CONFIGURATION; + } + } else if (argc < REQUIRED_ARGC[*version_ptr]) { + *error_ptr = "Incorrect number of arguments for version"; + return VDO_BAD_CONFIGURATION; + } + + if (*version_ptr != TABLE_VERSION) { + vdo_log_warning("Detected version mismatch between kernel module and tools kernel: %d, tool: %d", + TABLE_VERSION, *version_ptr); + vdo_log_warning("Please consider upgrading management tools to match kernel."); + } + return VDO_SUCCESS; +} + +/* Free a list of non-NULL string pointers, and then the list itself. */ +static void free_string_array(char **string_array) +{ + unsigned int offset; + + for (offset = 0; string_array[offset] != NULL; offset++) + vdo_free(string_array[offset]); + vdo_free(string_array); +} + +/* + * Split the input string into substrings, separated at occurrences of the indicated character, + * returning a null-terminated list of string pointers. + * + * The string pointers and the pointer array itself should both be freed with vdo_free() when no + * longer needed. This can be done with vdo_free_string_array (below) if the pointers in the array + * are not changed. Since the array and copied strings are allocated by this function, it may only + * be used in contexts where allocation is permitted. + * + * Empty substrings are not ignored; that is, returned substrings may be empty strings if the + * separator occurs twice in a row. + */ +static int split_string(const char *string, char separator, char ***substring_array_ptr) +{ + unsigned int current_substring = 0, substring_count = 1; + const char *s; + char **substrings; + int result; + ptrdiff_t length; + + for (s = string; *s != 0; s++) { + if (*s == separator) + substring_count++; + } + + result = vdo_allocate(substring_count + 1, char *, "string-splitting array", + &substrings); + if (result != VDO_SUCCESS) + return result; + + for (s = string; *s != 0; s++) { + if (*s == separator) { + ptrdiff_t length = s - string; + + result = vdo_allocate(length + 1, char, "split string", + &substrings[current_substring]); + if (result != VDO_SUCCESS) { + free_string_array(substrings); + return result; + } + /* + * Trailing NUL is already in place after allocation; deal with the zero or + * more non-NUL bytes in the string. + */ + if (length > 0) + memcpy(substrings[current_substring], string, length); + string = s + 1; + current_substring++; + BUG_ON(current_substring >= substring_count); + } + } + /* Process final string, with no trailing separator. */ + BUG_ON(current_substring != (substring_count - 1)); + length = strlen(string); + + result = vdo_allocate(length + 1, char, "split string", + &substrings[current_substring]); + if (result != VDO_SUCCESS) { + free_string_array(substrings); + return result; + } + memcpy(substrings[current_substring], string, length); + current_substring++; + /* substrings[current_substring] is NULL already */ + *substring_array_ptr = substrings; + return VDO_SUCCESS; +} + +/* + * Join the input substrings into one string, joined with the indicated character, returning a + * string. array_length is a bound on the number of valid elements in substring_array, in case it + * is not NULL-terminated. + */ +static int join_strings(char **substring_array, size_t array_length, char separator, + char **string_ptr) +{ + size_t string_length = 0; + size_t i; + int result; + char *output, *current_position; + + for (i = 0; (i < array_length) && (substring_array[i] != NULL); i++) + string_length += strlen(substring_array[i]) + 1; + + result = vdo_allocate(string_length, char, __func__, &output); + if (result != VDO_SUCCESS) + return result; + + current_position = &output[0]; + + for (i = 0; (i < array_length) && (substring_array[i] != NULL); i++) { + current_position = vdo_append_to_buffer(current_position, + output + string_length, "%s", + substring_array[i]); + *current_position = separator; + current_position++; + } + + /* We output one too many separators; replace the last with a zero byte. */ + if (current_position != output) + *(current_position - 1) = '\0'; + + *string_ptr = output; + return VDO_SUCCESS; +} + +/** + * parse_bool() - Parse a two-valued option into a bool. + * @bool_str: The string value to convert to a bool. + * @true_str: The string value which should be converted to true. + * @false_str: The string value which should be converted to false. + * @bool_ptr: A pointer to return the bool value in. + * + * Return: VDO_SUCCESS or an error if bool_str is neither true_str nor false_str. + */ +static inline int __must_check parse_bool(const char *bool_str, const char *true_str, + const char *false_str, bool *bool_ptr) +{ + bool value = false; + + if (strcmp(bool_str, true_str) == 0) + value = true; + else if (strcmp(bool_str, false_str) == 0) + value = false; + else + return VDO_BAD_CONFIGURATION; + + *bool_ptr = value; + return VDO_SUCCESS; +} + +/** + * process_one_thread_config_spec() - Process one component of a thread parameter configuration + * string and update the configuration data structure. + * @thread_param_type: The type of thread specified. + * @count: The thread count requested. + * @config: The configuration data structure to update. + * + * If the thread count requested is invalid, a message is logged and -EINVAL returned. If the + * thread name is unknown, a message is logged but no error is returned. + * + * Return: VDO_SUCCESS or -EINVAL + */ +static int process_one_thread_config_spec(const char *thread_param_type, + unsigned int count, + struct thread_count_config *config) +{ + /* Handle limited thread parameters */ + if (strcmp(thread_param_type, "bioRotationInterval") == 0) { + if (count == 0) { + vdo_log_error("thread config string error: 'bioRotationInterval' of at least 1 is required"); + return -EINVAL; + } else if (count > VDO_BIO_ROTATION_INTERVAL_LIMIT) { + vdo_log_error("thread config string error: 'bioRotationInterval' cannot be higher than %d", + VDO_BIO_ROTATION_INTERVAL_LIMIT); + return -EINVAL; + } + config->bio_rotation_interval = count; + return VDO_SUCCESS; + } + if (strcmp(thread_param_type, "logical") == 0) { + if (count > MAX_VDO_LOGICAL_ZONES) { + vdo_log_error("thread config string error: at most %d 'logical' threads are allowed", + MAX_VDO_LOGICAL_ZONES); + return -EINVAL; + } + config->logical_zones = count; + return VDO_SUCCESS; + } + if (strcmp(thread_param_type, "physical") == 0) { + if (count > MAX_VDO_PHYSICAL_ZONES) { + vdo_log_error("thread config string error: at most %d 'physical' threads are allowed", + MAX_VDO_PHYSICAL_ZONES); + return -EINVAL; + } + config->physical_zones = count; + return VDO_SUCCESS; + } + /* Handle other thread count parameters */ + if (count > MAXIMUM_VDO_THREADS) { + vdo_log_error("thread config string error: at most %d '%s' threads are allowed", + MAXIMUM_VDO_THREADS, thread_param_type); + return -EINVAL; + } + if (strcmp(thread_param_type, "hash") == 0) { + config->hash_zones = count; + return VDO_SUCCESS; + } + if (strcmp(thread_param_type, "cpu") == 0) { + if (count == 0) { + vdo_log_error("thread config string error: at least one 'cpu' thread required"); + return -EINVAL; + } + config->cpu_threads = count; + return VDO_SUCCESS; + } + if (strcmp(thread_param_type, "ack") == 0) { + config->bio_ack_threads = count; + return VDO_SUCCESS; + } + if (strcmp(thread_param_type, "bio") == 0) { + if (count == 0) { + vdo_log_error("thread config string error: at least one 'bio' thread required"); + return -EINVAL; + } + config->bio_threads = count; + return VDO_SUCCESS; + } + + /* + * Don't fail, just log. This will handle version mismatches between user mode tools and + * kernel. + */ + vdo_log_info("unknown thread parameter type \"%s\"", thread_param_type); + return VDO_SUCCESS; +} + +/** + * parse_one_thread_config_spec() - Parse one component of a thread parameter configuration string + * and update the configuration data structure. + * @spec: The thread parameter specification string. + * @config: The configuration data to be updated. + */ +static int parse_one_thread_config_spec(const char *spec, + struct thread_count_config *config) +{ + unsigned int count; + char **fields; + int result; + + result = split_string(spec, '=', &fields); + if (result != VDO_SUCCESS) + return result; + + if ((fields[0] == NULL) || (fields[1] == NULL) || (fields[2] != NULL)) { + vdo_log_error("thread config string error: expected thread parameter assignment, saw \"%s\"", + spec); + free_string_array(fields); + return -EINVAL; + } + + result = kstrtouint(fields[1], 10, &count); + if (result) { + vdo_log_error("thread config string error: integer value needed, found \"%s\"", + fields[1]); + free_string_array(fields); + return result; + } + + result = process_one_thread_config_spec(fields[0], count, config); + free_string_array(fields); + return result; +} + +/** + * parse_thread_config_string() - Parse the configuration string passed and update the specified + * counts and other parameters of various types of threads to be + * created. + * @string: Thread parameter configuration string. + * @config: The thread configuration data to update. + * + * The configuration string should contain one or more comma-separated specs of the form + * "typename=number"; the supported type names are "cpu", "ack", "bio", "bioRotationInterval", + * "logical", "physical", and "hash". + * + * If an error occurs during parsing of a single key/value pair, we deem it serious enough to stop + * further parsing. + * + * This function can't set the "reason" value the caller wants to pass back, because we'd want to + * format it to say which field was invalid, and we can't allocate the "reason" strings + * dynamically. So if an error occurs, we'll log the details and pass back an error. + * + * Return: VDO_SUCCESS or -EINVAL or -ENOMEM + */ +static int parse_thread_config_string(const char *string, + struct thread_count_config *config) +{ + int result = VDO_SUCCESS; + char **specs; + + if (strcmp(".", string) != 0) { + unsigned int i; + + result = split_string(string, ',', &specs); + if (result != VDO_SUCCESS) + return result; + + for (i = 0; specs[i] != NULL; i++) { + result = parse_one_thread_config_spec(specs[i], config); + if (result != VDO_SUCCESS) + break; + } + free_string_array(specs); + } + return result; +} + +/** + * process_one_key_value_pair() - Process one component of an optional parameter string and update + * the configuration data structure. + * @key: The optional parameter key name. + * @value: The optional parameter value. + * @config: The configuration data structure to update. + * + * If the value requested is invalid, a message is logged and -EINVAL returned. If the key is + * unknown, a message is logged but no error is returned. + * + * Return: VDO_SUCCESS or -EINVAL + */ +static int process_one_key_value_pair(const char *key, unsigned int value, + struct device_config *config) +{ + /* Non thread optional parameters */ + if (strcmp(key, "maxDiscard") == 0) { + if (value == 0) { + vdo_log_error("optional parameter error: at least one max discard block required"); + return -EINVAL; + } + /* Max discard sectors in blkdev_issue_discard is UINT_MAX >> 9 */ + if (value > (UINT_MAX / VDO_BLOCK_SIZE)) { + vdo_log_error("optional parameter error: at most %d max discard blocks are allowed", + UINT_MAX / VDO_BLOCK_SIZE); + return -EINVAL; + } + config->max_discard_blocks = value; + return VDO_SUCCESS; + } + /* Handles unknown key names */ + return process_one_thread_config_spec(key, value, &config->thread_counts); +} + +/** + * parse_one_key_value_pair() - Parse one key/value pair and update the configuration data + * structure. + * @key: The optional key name. + * @value: The optional value. + * @config: The configuration data to be updated. + * + * Return: VDO_SUCCESS or error. + */ +static int parse_one_key_value_pair(const char *key, const char *value, + struct device_config *config) +{ + unsigned int count; + int result; + + if (strcmp(key, "deduplication") == 0) + return parse_bool(value, "on", "off", &config->deduplication); + + if (strcmp(key, "compression") == 0) + return parse_bool(value, "on", "off", &config->compression); + + /* The remaining arguments must have integral values. */ + result = kstrtouint(value, 10, &count); + if (result) { + vdo_log_error("optional config string error: integer value needed, found \"%s\"", + value); + return result; + } + return process_one_key_value_pair(key, count, config); +} + +/** + * parse_key_value_pairs() - Parse all key/value pairs from a list of arguments. + * @argc: The total number of arguments in list. + * @argv: The list of key/value pairs. + * @config: The device configuration data to update. + * + * If an error occurs during parsing of a single key/value pair, we deem it serious enough to stop + * further parsing. + * + * This function can't set the "reason" value the caller wants to pass back, because we'd want to + * format it to say which field was invalid, and we can't allocate the "reason" strings + * dynamically. So if an error occurs, we'll log the details and return the error. + * + * Return: VDO_SUCCESS or error + */ +static int parse_key_value_pairs(int argc, char **argv, struct device_config *config) +{ + int result = VDO_SUCCESS; + + while (argc) { + result = parse_one_key_value_pair(argv[0], argv[1], config); + if (result != VDO_SUCCESS) + break; + + argc -= 2; + argv += 2; + } + + return result; +} + +/** + * parse_optional_arguments() - Parse the configuration string passed in for optional arguments. + * @arg_set: The structure holding the arguments to parse. + * @error_ptr: Pointer to a buffer to hold the error string. + * @config: Pointer to device configuration data to update. + * + * For V0/V1 configurations, there will only be one optional parameter; the thread configuration. + * The configuration string should contain one or more comma-separated specs of the form + * "typename=number"; the supported type names are "cpu", "ack", "bio", "bioRotationInterval", + * "logical", "physical", and "hash". + * + * For V2 configurations and beyond, there could be any number of arguments. They should contain + * one or more key/value pairs separated by a space. + * + * Return: VDO_SUCCESS or error + */ +static int parse_optional_arguments(struct dm_arg_set *arg_set, char **error_ptr, + struct device_config *config) +{ + int result = VDO_SUCCESS; + + if (config->version == 0 || config->version == 1) { + result = parse_thread_config_string(arg_set->argv[0], + &config->thread_counts); + if (result != VDO_SUCCESS) { + *error_ptr = "Invalid thread-count configuration"; + return VDO_BAD_CONFIGURATION; + } + } else { + if ((arg_set->argc % 2) != 0) { + *error_ptr = "Odd number of optional arguments given but they should be pairs"; + return VDO_BAD_CONFIGURATION; + } + result = parse_key_value_pairs(arg_set->argc, arg_set->argv, config); + if (result != VDO_SUCCESS) { + *error_ptr = "Invalid optional argument configuration"; + return VDO_BAD_CONFIGURATION; + } + } + return result; +} + +/** + * handle_parse_error() - Handle a parsing error. + * @config: The config to free. + * @error_ptr: A place to store a constant string about the error. + * @error_str: A constant string to store in error_ptr. + */ +static void handle_parse_error(struct device_config *config, char **error_ptr, + char *error_str) +{ + free_device_config(config); + *error_ptr = error_str; +} + +/** + * parse_device_config() - Convert the dmsetup table into a struct device_config. + * @argc: The number of table values. + * @argv: The array of table values. + * @ti: The target structure for this table. + * @config_ptr: A pointer to return the allocated config. + * + * Return: VDO_SUCCESS or an error code. + */ +static int parse_device_config(int argc, char **argv, struct dm_target *ti, + struct device_config **config_ptr) +{ + bool enable_512e; + size_t logical_bytes = to_bytes(ti->len); + struct dm_arg_set arg_set; + char **error_ptr = &ti->error; + struct device_config *config = NULL; + int result; + + if ((logical_bytes % VDO_BLOCK_SIZE) != 0) { + handle_parse_error(config, error_ptr, + "Logical size must be a multiple of 4096"); + return VDO_BAD_CONFIGURATION; + } + + if (argc == 0) { + handle_parse_error(config, error_ptr, "Incorrect number of arguments"); + return VDO_BAD_CONFIGURATION; + } + + result = vdo_allocate(1, struct device_config, "device_config", &config); + if (result != VDO_SUCCESS) { + handle_parse_error(config, error_ptr, + "Could not allocate config structure"); + return VDO_BAD_CONFIGURATION; + } + + config->owning_target = ti; + config->logical_blocks = logical_bytes / VDO_BLOCK_SIZE; + INIT_LIST_HEAD(&config->config_list); + + /* Save the original string. */ + result = join_strings(argv, argc, ' ', &config->original_string); + if (result != VDO_SUCCESS) { + handle_parse_error(config, error_ptr, "Could not populate string"); + return VDO_BAD_CONFIGURATION; + } + + vdo_log_info("table line: %s", config->original_string); + + config->thread_counts = (struct thread_count_config) { + .bio_ack_threads = 1, + .bio_threads = DEFAULT_VDO_BIO_SUBMIT_QUEUE_COUNT, + .bio_rotation_interval = DEFAULT_VDO_BIO_SUBMIT_QUEUE_ROTATE_INTERVAL, + .cpu_threads = 1, + .logical_zones = 0, + .physical_zones = 0, + .hash_zones = 0, + }; + config->max_discard_blocks = 1; + config->deduplication = true; + config->compression = false; + + arg_set.argc = argc; + arg_set.argv = argv; + + result = get_version_number(argc, argv, error_ptr, &config->version); + if (result != VDO_SUCCESS) { + /* get_version_number sets error_ptr itself. */ + handle_parse_error(config, error_ptr, *error_ptr); + return result; + } + /* Move the arg pointer forward only if the argument was there. */ + if (config->version >= 1) + dm_shift_arg(&arg_set); + + result = vdo_duplicate_string(dm_shift_arg(&arg_set), "parent device name", + &config->parent_device_name); + if (result != VDO_SUCCESS) { + handle_parse_error(config, error_ptr, + "Could not copy parent device name"); + return VDO_BAD_CONFIGURATION; + } + + /* Get the physical blocks, if known. */ + if (config->version >= 1) { + result = kstrtoull(dm_shift_arg(&arg_set), 10, &config->physical_blocks); + if (result != VDO_SUCCESS) { + handle_parse_error(config, error_ptr, + "Invalid physical block count"); + return VDO_BAD_CONFIGURATION; + } + } + + /* Get the logical block size and validate */ + result = parse_bool(dm_shift_arg(&arg_set), "512", "4096", &enable_512e); + if (result != VDO_SUCCESS) { + handle_parse_error(config, error_ptr, "Invalid logical block size"); + return VDO_BAD_CONFIGURATION; + } + config->logical_block_size = (enable_512e ? 512 : 4096); + + /* Skip past the two no longer used read cache options. */ + if (config->version <= 1) + dm_consume_args(&arg_set, 2); + + /* Get the page cache size. */ + result = kstrtouint(dm_shift_arg(&arg_set), 10, &config->cache_size); + if (result != VDO_SUCCESS) { + handle_parse_error(config, error_ptr, + "Invalid block map page cache size"); + return VDO_BAD_CONFIGURATION; + } + + /* Get the block map era length. */ + result = kstrtouint(dm_shift_arg(&arg_set), 10, &config->block_map_maximum_age); + if (result != VDO_SUCCESS) { + handle_parse_error(config, error_ptr, "Invalid block map maximum age"); + return VDO_BAD_CONFIGURATION; + } + + /* Skip past the no longer used MD RAID5 optimization mode */ + if (config->version <= 2) + dm_consume_args(&arg_set, 1); + + /* Skip past the no longer used write policy setting */ + if (config->version <= 3) + dm_consume_args(&arg_set, 1); + + /* Skip past the no longer used pool name for older table lines */ + if (config->version <= 2) { + /* + * Make sure the enum to get the pool name from argv directly is still in sync with + * the parsing of the table line. + */ + if (&arg_set.argv[0] != &argv[POOL_NAME_ARG_INDEX[config->version]]) { + handle_parse_error(config, error_ptr, + "Pool name not in expected location"); + return VDO_BAD_CONFIGURATION; + } + dm_shift_arg(&arg_set); + } + + /* Get the optional arguments and validate. */ + result = parse_optional_arguments(&arg_set, error_ptr, config); + if (result != VDO_SUCCESS) { + /* parse_optional_arguments sets error_ptr itself. */ + handle_parse_error(config, error_ptr, *error_ptr); + return result; + } + + /* + * Logical, physical, and hash zone counts can all be zero; then we get one thread doing + * everything, our older configuration. If any zone count is non-zero, the others must be + * as well. + */ + if (((config->thread_counts.logical_zones == 0) != + (config->thread_counts.physical_zones == 0)) || + ((config->thread_counts.physical_zones == 0) != + (config->thread_counts.hash_zones == 0))) { + handle_parse_error(config, error_ptr, + "Logical, physical, and hash zones counts must all be zero or all non-zero"); + return VDO_BAD_CONFIGURATION; + } + + if (config->cache_size < + (2 * MAXIMUM_VDO_USER_VIOS * config->thread_counts.logical_zones)) { + handle_parse_error(config, error_ptr, + "Insufficient block map cache for logical zones"); + return VDO_BAD_CONFIGURATION; + } + + result = dm_get_device(ti, config->parent_device_name, + dm_table_get_mode(ti->table), &config->owned_device); + if (result != 0) { + vdo_log_error("couldn't open device \"%s\": error %d", + config->parent_device_name, result); + handle_parse_error(config, error_ptr, "Unable to open storage device"); + return VDO_BAD_CONFIGURATION; + } + + if (config->version == 0) { + u64 device_size = i_size_read(config->owned_device->bdev->bd_inode); + + config->physical_blocks = device_size / VDO_BLOCK_SIZE; + } + + *config_ptr = config; + return result; +} + +static struct vdo *get_vdo_for_target(struct dm_target *ti) +{ + return ((struct device_config *) ti->private)->vdo; +} + + +static int vdo_map_bio(struct dm_target *ti, struct bio *bio) +{ + struct vdo *vdo = get_vdo_for_target(ti); + struct vdo_work_queue *current_work_queue; + const struct admin_state_code *code = vdo_get_admin_state_code(&vdo->admin.state); + + VDO_ASSERT_LOG_ONLY(code->normal, "vdo should not receive bios while in state %s", + code->name); + + /* Count all incoming bios. */ + vdo_count_bios(&vdo->stats.bios_in, bio); + + + /* Handle empty bios. Empty flush bios are not associated with a vio. */ + if ((bio_op(bio) == REQ_OP_FLUSH) || ((bio->bi_opf & REQ_PREFLUSH) != 0)) { + vdo_launch_flush(vdo, bio); + return DM_MAPIO_SUBMITTED; + } + + /* This could deadlock, */ + current_work_queue = vdo_get_current_work_queue(); + BUG_ON((current_work_queue != NULL) && + (vdo == vdo_get_work_queue_owner(current_work_queue)->vdo)); + vdo_launch_bio(vdo->data_vio_pool, bio); + return DM_MAPIO_SUBMITTED; +} + +static void vdo_io_hints(struct dm_target *ti, struct queue_limits *limits) +{ + struct vdo *vdo = get_vdo_for_target(ti); + + limits->logical_block_size = vdo->device_config->logical_block_size; + limits->physical_block_size = VDO_BLOCK_SIZE; + + /* The minimum io size for random io */ + blk_limits_io_min(limits, VDO_BLOCK_SIZE); + /* The optimal io size for streamed/sequential io */ + blk_limits_io_opt(limits, VDO_BLOCK_SIZE); + + /* + * Sets the maximum discard size that will be passed into VDO. This value comes from a + * table line value passed in during dmsetup create. + * + * The value 1024 is the largest usable value on HD systems. A 2048 sector discard on a + * busy HD system takes 31 seconds. We should use a value no higher than 1024, which takes + * 15 to 16 seconds on a busy HD system. However, using large values results in 120 second + * blocked task warnings in kernel logs. In order to avoid these warnings, we choose to + * use the smallest reasonable value. + * + * The value is used by dm-thin to determine whether to pass down discards. The block layer + * splits large discards on this boundary when this is set. + */ + limits->max_discard_sectors = + (vdo->device_config->max_discard_blocks * VDO_SECTORS_PER_BLOCK); + + /* + * Force discards to not begin or end with a partial block by stating the granularity is + * 4k. + */ + limits->discard_granularity = VDO_BLOCK_SIZE; +} + +static int vdo_iterate_devices(struct dm_target *ti, iterate_devices_callout_fn fn, + void *data) +{ + struct device_config *config = get_vdo_for_target(ti)->device_config; + + return fn(ti, config->owned_device, 0, + config->physical_blocks * VDO_SECTORS_PER_BLOCK, data); +} + +/* + * Status line is: + * + * + */ + +static void vdo_status(struct dm_target *ti, status_type_t status_type, + unsigned int status_flags, char *result, unsigned int maxlen) +{ + struct vdo *vdo = get_vdo_for_target(ti); + struct vdo_statistics *stats; + struct device_config *device_config; + /* N.B.: The DMEMIT macro uses the variables named "sz", "result", "maxlen". */ + int sz = 0; + + switch (status_type) { + case STATUSTYPE_INFO: + /* Report info for dmsetup status */ + mutex_lock(&vdo->stats_mutex); + vdo_fetch_statistics(vdo, &vdo->stats_buffer); + stats = &vdo->stats_buffer; + + DMEMIT("/dev/%pg %s %s %s %s %llu %llu", + vdo_get_backing_device(vdo), stats->mode, + stats->in_recovery_mode ? "recovering" : "-", + vdo_get_dedupe_index_state_name(vdo->hash_zones), + vdo_get_compressing(vdo) ? "online" : "offline", + stats->data_blocks_used + stats->overhead_blocks_used, + stats->physical_blocks); + mutex_unlock(&vdo->stats_mutex); + break; + + case STATUSTYPE_TABLE: + /* Report the string actually specified in the beginning. */ + device_config = (struct device_config *) ti->private; + DMEMIT("%s", device_config->original_string); + break; + + case STATUSTYPE_IMA: + /* FIXME: We ought to be more detailed here, but this is what thin does. */ + *result = '\0'; + break; + } +} + +static block_count_t __must_check get_underlying_device_block_count(const struct vdo *vdo) +{ + return i_size_read(vdo_get_backing_device(vdo)->bd_inode) / VDO_BLOCK_SIZE; +} + +static int __must_check process_vdo_message_locked(struct vdo *vdo, unsigned int argc, + char **argv) +{ + if ((argc == 2) && (strcasecmp(argv[0], "compression") == 0)) { + if (strcasecmp(argv[1], "on") == 0) { + vdo_set_compressing(vdo, true); + return 0; + } + + if (strcasecmp(argv[1], "off") == 0) { + vdo_set_compressing(vdo, false); + return 0; + } + + vdo_log_warning("invalid argument '%s' to dmsetup compression message", + argv[1]); + return -EINVAL; + } + + vdo_log_warning("unrecognized dmsetup message '%s' received", argv[0]); + return -EINVAL; +} + +/* + * If the message is a dump, just do it. Otherwise, check that no other message is being processed, + * and only proceed if so. + * Returns -EBUSY if another message is being processed + */ +static int __must_check process_vdo_message(struct vdo *vdo, unsigned int argc, + char **argv) +{ + int result; + + /* + * All messages which may be processed in parallel with other messages should be handled + * here before the atomic check below. Messages which should be exclusive should be + * processed in process_vdo_message_locked(). + */ + + /* Dump messages should always be processed */ + if (strcasecmp(argv[0], "dump") == 0) + return vdo_dump(vdo, argc, argv, "dmsetup message"); + + if (argc == 1) { + if (strcasecmp(argv[0], "dump-on-shutdown") == 0) { + vdo->dump_on_shutdown = true; + return 0; + } + + /* Index messages should always be processed */ + if ((strcasecmp(argv[0], "index-close") == 0) || + (strcasecmp(argv[0], "index-create") == 0) || + (strcasecmp(argv[0], "index-disable") == 0) || + (strcasecmp(argv[0], "index-enable") == 0)) + return vdo_message_dedupe_index(vdo->hash_zones, argv[0]); + } + + if (atomic_cmpxchg(&vdo->processing_message, 0, 1) != 0) + return -EBUSY; + + result = process_vdo_message_locked(vdo, argc, argv); + + /* Pairs with the implicit barrier in cmpxchg just above */ + smp_wmb(); + atomic_set(&vdo->processing_message, 0); + return result; +} + +static int vdo_message(struct dm_target *ti, unsigned int argc, char **argv, + char *result_buffer, unsigned int maxlen) +{ + struct registered_thread allocating_thread, instance_thread; + struct vdo *vdo; + int result; + + if (argc == 0) { + vdo_log_warning("unspecified dmsetup message"); + return -EINVAL; + } + + vdo = get_vdo_for_target(ti); + vdo_register_allocating_thread(&allocating_thread, NULL); + vdo_register_thread_device_id(&instance_thread, &vdo->instance); + + /* + * Must be done here so we don't map return codes. The code in dm-ioctl expects a 1 for a + * return code to look at the buffer and see if it is full or not. + */ + if ((argc == 1) && (strcasecmp(argv[0], "stats") == 0)) { + vdo_write_stats(vdo, result_buffer, maxlen); + result = 1; + } else { + result = vdo_status_to_errno(process_vdo_message(vdo, argc, argv)); + } + + vdo_unregister_thread_device_id(); + vdo_unregister_allocating_thread(); + return result; +} + +static void configure_target_capabilities(struct dm_target *ti) +{ + ti->discards_supported = 1; + ti->flush_supported = true; + ti->num_discard_bios = 1; + ti->num_flush_bios = 1; + + /* + * If this value changes, please make sure to update the value for max_discard_sectors + * accordingly. + */ + BUG_ON(dm_set_target_max_io_len(ti, VDO_SECTORS_PER_BLOCK) != 0); +} + +/* + * Implements vdo_filter_fn. + */ +static bool vdo_uses_device(struct vdo *vdo, const void *context) +{ + const struct device_config *config = context; + + return vdo_get_backing_device(vdo)->bd_dev == config->owned_device->bdev->bd_dev; +} + +/** + * get_thread_id_for_phase() - Get the thread id for the current phase of the admin operation in + * progress. + */ +static thread_id_t __must_check get_thread_id_for_phase(struct vdo *vdo) +{ + switch (vdo->admin.phase) { + case RESUME_PHASE_PACKER: + case RESUME_PHASE_FLUSHER: + case SUSPEND_PHASE_PACKER: + case SUSPEND_PHASE_FLUSHES: + return vdo->thread_config.packer_thread; + + case RESUME_PHASE_DATA_VIOS: + case SUSPEND_PHASE_DATA_VIOS: + return vdo->thread_config.cpu_thread; + + case LOAD_PHASE_DRAIN_JOURNAL: + case RESUME_PHASE_JOURNAL: + case SUSPEND_PHASE_JOURNAL: + return vdo->thread_config.journal_thread; + + default: + return vdo->thread_config.admin_thread; + } +} + +static struct vdo_completion *prepare_admin_completion(struct vdo *vdo, + vdo_action_fn callback, + vdo_action_fn error_handler) +{ + struct vdo_completion *completion = &vdo->admin.completion; + + /* + * We can't use vdo_prepare_completion_for_requeue() here because we don't want to reset + * any error in the completion. + */ + completion->callback = callback; + completion->error_handler = error_handler; + completion->callback_thread_id = get_thread_id_for_phase(vdo); + completion->requeue = true; + return completion; +} + +/** + * advance_phase() - Increment the phase of the current admin operation and prepare the admin + * completion to run on the thread for the next phase. + * @vdo: The on which an admin operation is being performed + * + * Return: The current phase + */ +static u32 advance_phase(struct vdo *vdo) +{ + u32 phase = vdo->admin.phase++; + + vdo->admin.completion.callback_thread_id = get_thread_id_for_phase(vdo); + vdo->admin.completion.requeue = true; + return phase; +} + +/* + * Perform an administrative operation (load, suspend, grow logical, or grow physical). This method + * should not be called from vdo threads. + */ +static int perform_admin_operation(struct vdo *vdo, u32 starting_phase, + vdo_action_fn callback, vdo_action_fn error_handler, + const char *type) +{ + int result; + struct vdo_administrator *admin = &vdo->admin; + + if (atomic_cmpxchg(&admin->busy, 0, 1) != 0) { + return vdo_log_error_strerror(VDO_COMPONENT_BUSY, + "Can't start %s operation, another operation is already in progress", + type); + } + + admin->phase = starting_phase; + reinit_completion(&admin->callback_sync); + vdo_reset_completion(&admin->completion); + vdo_launch_completion(prepare_admin_completion(vdo, callback, error_handler)); + + /* + * Using the "interruptible" interface means that Linux will not log a message when we wait + * for more than 120 seconds. + */ + while (wait_for_completion_interruptible(&admin->callback_sync)) { + /* However, if we get a signal in a user-mode process, we could spin... */ + fsleep(1000); + } + + result = admin->completion.result; + /* pairs with implicit barrier in cmpxchg above */ + smp_wmb(); + atomic_set(&admin->busy, 0); + return result; +} + +/* Assert that we are operating on the correct thread for the current phase. */ +static void assert_admin_phase_thread(struct vdo *vdo, const char *what) +{ + VDO_ASSERT_LOG_ONLY(vdo_get_callback_thread_id() == get_thread_id_for_phase(vdo), + "%s on correct thread for %s", what, + ADMIN_PHASE_NAMES[vdo->admin.phase]); +} + +/** + * finish_operation_callback() - Callback to finish an admin operation. + * @completion: The admin_completion. + */ +static void finish_operation_callback(struct vdo_completion *completion) +{ + struct vdo_administrator *admin = &completion->vdo->admin; + + vdo_finish_operation(&admin->state, completion->result); + complete(&admin->callback_sync); +} + +/** + * decode_from_super_block() - Decode the VDO state from the super block and validate that it is + * correct. + * @vdo: The vdo being loaded. + * + * On error from this method, the component states must be destroyed explicitly. If this method + * returns successfully, the component states must not be destroyed. + * + * Return: VDO_SUCCESS or an error. + */ +static int __must_check decode_from_super_block(struct vdo *vdo) +{ + const struct device_config *config = vdo->device_config; + int result; + + result = vdo_decode_component_states(vdo->super_block.buffer, &vdo->geometry, + &vdo->states); + if (result != VDO_SUCCESS) + return result; + + vdo_set_state(vdo, vdo->states.vdo.state); + vdo->load_state = vdo->states.vdo.state; + + /* + * If the device config specifies a larger logical size than was recorded in the super + * block, just accept it. + */ + if (vdo->states.vdo.config.logical_blocks < config->logical_blocks) { + vdo_log_warning("Growing logical size: a logical size of %llu blocks was specified, but that differs from the %llu blocks configured in the vdo super block", + (unsigned long long) config->logical_blocks, + (unsigned long long) vdo->states.vdo.config.logical_blocks); + vdo->states.vdo.config.logical_blocks = config->logical_blocks; + } + + result = vdo_validate_component_states(&vdo->states, vdo->geometry.nonce, + config->physical_blocks, + config->logical_blocks); + if (result != VDO_SUCCESS) + return result; + + vdo->layout = vdo->states.layout; + return VDO_SUCCESS; +} + +/** + * decode_vdo() - Decode the component data portion of a super block and fill in the corresponding + * portions of the vdo being loaded. + * @vdo: The vdo being loaded. + * + * This will also allocate the recovery journal and slab depot. If this method is called with an + * asynchronous layer (i.e. a thread config which specifies at least one base thread), the block + * map and packer will be constructed as well. + * + * Return: VDO_SUCCESS or an error. + */ +static int __must_check decode_vdo(struct vdo *vdo) +{ + block_count_t maximum_age, journal_length; + struct partition *partition; + int result; + + result = decode_from_super_block(vdo); + if (result != VDO_SUCCESS) { + vdo_destroy_component_states(&vdo->states); + return result; + } + + maximum_age = vdo_convert_maximum_age(vdo->device_config->block_map_maximum_age); + journal_length = + vdo_get_recovery_journal_length(vdo->states.vdo.config.recovery_journal_size); + if (maximum_age > (journal_length / 2)) { + return vdo_log_error_strerror(VDO_BAD_CONFIGURATION, + "maximum age: %llu exceeds limit %llu", + (unsigned long long) maximum_age, + (unsigned long long) (journal_length / 2)); + } + + if (maximum_age == 0) { + return vdo_log_error_strerror(VDO_BAD_CONFIGURATION, + "maximum age must be greater than 0"); + } + + result = vdo_enable_read_only_entry(vdo); + if (result != VDO_SUCCESS) + return result; + + partition = vdo_get_known_partition(&vdo->layout, + VDO_RECOVERY_JOURNAL_PARTITION); + result = vdo_decode_recovery_journal(vdo->states.recovery_journal, + vdo->states.vdo.nonce, vdo, partition, + vdo->states.vdo.complete_recoveries, + vdo->states.vdo.config.recovery_journal_size, + &vdo->recovery_journal); + if (result != VDO_SUCCESS) + return result; + + partition = vdo_get_known_partition(&vdo->layout, VDO_SLAB_SUMMARY_PARTITION); + result = vdo_decode_slab_depot(vdo->states.slab_depot, vdo, partition, + &vdo->depot); + if (result != VDO_SUCCESS) + return result; + + result = vdo_decode_block_map(vdo->states.block_map, + vdo->states.vdo.config.logical_blocks, vdo, + vdo->recovery_journal, vdo->states.vdo.nonce, + vdo->device_config->cache_size, maximum_age, + &vdo->block_map); + if (result != VDO_SUCCESS) + return result; + + result = vdo_make_physical_zones(vdo, &vdo->physical_zones); + if (result != VDO_SUCCESS) + return result; + + /* The logical zones depend on the physical zones already existing. */ + result = vdo_make_logical_zones(vdo, &vdo->logical_zones); + if (result != VDO_SUCCESS) + return result; + + return vdo_make_hash_zones(vdo, &vdo->hash_zones); +} + +/** + * pre_load_callback() - Callback to initiate a pre-load, registered in vdo_initialize(). + * @completion: The admin completion. + */ +static void pre_load_callback(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + int result; + + assert_admin_phase_thread(vdo, __func__); + + switch (advance_phase(vdo)) { + case PRE_LOAD_PHASE_START: + result = vdo_start_operation(&vdo->admin.state, + VDO_ADMIN_STATE_PRE_LOADING); + if (result != VDO_SUCCESS) { + vdo_continue_completion(completion, result); + return; + } + + vdo_load_super_block(vdo, completion); + return; + + case PRE_LOAD_PHASE_LOAD_COMPONENTS: + vdo_continue_completion(completion, decode_vdo(vdo)); + return; + + case PRE_LOAD_PHASE_END: + break; + + default: + vdo_set_completion_result(completion, UDS_BAD_STATE); + } + + finish_operation_callback(completion); +} + +static void release_instance(unsigned int instance) +{ + mutex_lock(&instances_lock); + if (instance >= instances.bit_count) { + VDO_ASSERT_LOG_ONLY(false, + "instance number %u must be less than bit count %u", + instance, instances.bit_count); + } else if (test_bit(instance, instances.words) == 0) { + VDO_ASSERT_LOG_ONLY(false, "instance number %u must be allocated", instance); + } else { + __clear_bit(instance, instances.words); + instances.count -= 1; + } + mutex_unlock(&instances_lock); +} + +static void set_device_config(struct dm_target *ti, struct vdo *vdo, + struct device_config *config) +{ + list_del_init(&config->config_list); + list_add_tail(&config->config_list, &vdo->device_config_list); + config->vdo = vdo; + ti->private = config; + configure_target_capabilities(ti); +} + +static int vdo_initialize(struct dm_target *ti, unsigned int instance, + struct device_config *config) +{ + struct vdo *vdo; + int result; + u64 block_size = VDO_BLOCK_SIZE; + u64 logical_size = to_bytes(ti->len); + block_count_t logical_blocks = logical_size / block_size; + + vdo_log_info("loading device '%s'", vdo_get_device_name(ti)); + vdo_log_debug("Logical block size = %llu", (u64) config->logical_block_size); + vdo_log_debug("Logical blocks = %llu", logical_blocks); + vdo_log_debug("Physical block size = %llu", (u64) block_size); + vdo_log_debug("Physical blocks = %llu", config->physical_blocks); + vdo_log_debug("Block map cache blocks = %u", config->cache_size); + vdo_log_debug("Block map maximum age = %u", config->block_map_maximum_age); + vdo_log_debug("Deduplication = %s", (config->deduplication ? "on" : "off")); + vdo_log_debug("Compression = %s", (config->compression ? "on" : "off")); + + vdo = vdo_find_matching(vdo_uses_device, config); + if (vdo != NULL) { + vdo_log_error("Existing vdo already uses device %s", + vdo->device_config->parent_device_name); + ti->error = "Cannot share storage device with already-running VDO"; + return VDO_BAD_CONFIGURATION; + } + + result = vdo_make(instance, config, &ti->error, &vdo); + if (result != VDO_SUCCESS) { + vdo_log_error("Could not create VDO device. (VDO error %d, message %s)", + result, ti->error); + vdo_destroy(vdo); + return result; + } + + result = perform_admin_operation(vdo, PRE_LOAD_PHASE_START, pre_load_callback, + finish_operation_callback, "pre-load"); + if (result != VDO_SUCCESS) { + ti->error = ((result == VDO_INVALID_ADMIN_STATE) ? + "Pre-load is only valid immediately after initialization" : + "Cannot load metadata from device"); + vdo_log_error("Could not start VDO device. (VDO error %d, message %s)", + result, ti->error); + vdo_destroy(vdo); + return result; + } + + set_device_config(ti, vdo, config); + vdo->device_config = config; + return VDO_SUCCESS; +} + +/* Implements vdo_filter_fn. */ +static bool __must_check vdo_is_named(struct vdo *vdo, const void *context) +{ + struct dm_target *ti = vdo->device_config->owning_target; + const char *device_name = vdo_get_device_name(ti); + + return strcmp(device_name, context) == 0; +} + +/** + * get_bit_array_size() - Return the number of bytes needed to store a bit array of the specified + * capacity in an array of unsigned longs. + * @bit_count: The number of bits the array must hold. + * + * Return: the number of bytes needed for the array representation. + */ +static size_t get_bit_array_size(unsigned int bit_count) +{ + /* Round up to a multiple of the word size and convert to a byte count. */ + return (BITS_TO_LONGS(bit_count) * sizeof(unsigned long)); +} + +/** + * grow_bit_array() - Re-allocate the bitmap word array so there will more instance numbers that + * can be allocated. + * + * Since the array is initially NULL, this also initializes the array the first time we allocate an + * instance number. + * + * Return: VDO_SUCCESS or an error code from the allocation + */ +static int grow_bit_array(void) +{ + unsigned int new_count = max(instances.bit_count + BIT_COUNT_INCREMENT, + (unsigned int) BIT_COUNT_MINIMUM); + unsigned long *new_words; + int result; + + result = vdo_reallocate_memory(instances.words, + get_bit_array_size(instances.bit_count), + get_bit_array_size(new_count), + "instance number bit array", &new_words); + if (result != VDO_SUCCESS) + return result; + + instances.bit_count = new_count; + instances.words = new_words; + return VDO_SUCCESS; +} + +/** + * allocate_instance() - Allocate an instance number. + * @instance_ptr: A point to hold the instance number + * + * Return: VDO_SUCCESS or an error code + * + * This function must be called while holding the instances lock. + */ +static int allocate_instance(unsigned int *instance_ptr) +{ + unsigned int instance; + int result; + + /* If there are no unallocated instances, grow the bit array. */ + if (instances.count >= instances.bit_count) { + result = grow_bit_array(); + if (result != VDO_SUCCESS) + return result; + } + + /* + * There must be a zero bit somewhere now. Find it, starting just after the last instance + * allocated. + */ + instance = find_next_zero_bit(instances.words, instances.bit_count, + instances.next); + if (instance >= instances.bit_count) { + /* Nothing free after next, so wrap around to instance zero. */ + instance = find_first_zero_bit(instances.words, instances.bit_count); + result = VDO_ASSERT(instance < instances.bit_count, + "impossibly, no zero bit found"); + if (result != VDO_SUCCESS) + return result; + } + + __set_bit(instance, instances.words); + instances.count++; + instances.next = instance + 1; + *instance_ptr = instance; + return VDO_SUCCESS; +} + +static int construct_new_vdo_registered(struct dm_target *ti, unsigned int argc, + char **argv, unsigned int instance) +{ + int result; + struct device_config *config; + + result = parse_device_config(argc, argv, ti, &config); + if (result != VDO_SUCCESS) { + vdo_log_error_strerror(result, "parsing failed: %s", ti->error); + release_instance(instance); + return -EINVAL; + } + + /* Beyond this point, the instance number will be cleaned up for us if needed */ + result = vdo_initialize(ti, instance, config); + if (result != VDO_SUCCESS) { + release_instance(instance); + free_device_config(config); + return vdo_status_to_errno(result); + } + + return VDO_SUCCESS; +} + +static int construct_new_vdo(struct dm_target *ti, unsigned int argc, char **argv) +{ + int result; + unsigned int instance; + struct registered_thread instance_thread; + + mutex_lock(&instances_lock); + result = allocate_instance(&instance); + mutex_unlock(&instances_lock); + if (result != VDO_SUCCESS) + return -ENOMEM; + + vdo_register_thread_device_id(&instance_thread, &instance); + result = construct_new_vdo_registered(ti, argc, argv, instance); + vdo_unregister_thread_device_id(); + return result; +} + +/** + * check_may_grow_physical() - Callback to check that we're not in recovery mode, used in + * vdo_prepare_to_grow_physical(). + * @completion: The admin completion. + */ +static void check_may_grow_physical(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + + assert_admin_phase_thread(vdo, __func__); + + /* These checks can only be done from a vdo thread. */ + if (vdo_is_read_only(vdo)) + vdo_set_completion_result(completion, VDO_READ_ONLY); + + if (vdo_in_recovery_mode(vdo)) + vdo_set_completion_result(completion, VDO_RETRY_AFTER_REBUILD); + + finish_operation_callback(completion); +} + +static block_count_t get_partition_size(struct layout *layout, enum partition_id id) +{ + return vdo_get_known_partition(layout, id)->count; +} + +/** + * grow_layout() - Make the layout for growing a vdo. + * @vdo: The vdo preparing to grow. + * @old_size: The current size of the vdo. + * @new_size: The size to which the vdo will be grown. + * + * Return: VDO_SUCCESS or an error code. + */ +static int grow_layout(struct vdo *vdo, block_count_t old_size, block_count_t new_size) +{ + int result; + block_count_t min_new_size; + + if (vdo->next_layout.size == new_size) { + /* We are already prepared to grow to the new size, so we're done. */ + return VDO_SUCCESS; + } + + /* Make a copy completion if there isn't one */ + if (vdo->partition_copier == NULL) { + vdo->partition_copier = dm_kcopyd_client_create(NULL); + if (IS_ERR(vdo->partition_copier)) { + result = PTR_ERR(vdo->partition_copier); + vdo->partition_copier = NULL; + return result; + } + } + + /* Free any unused preparation. */ + vdo_uninitialize_layout(&vdo->next_layout); + + /* + * Make a new layout with the existing partition sizes for everything but the slab depot + * partition. + */ + result = vdo_initialize_layout(new_size, vdo->layout.start, + get_partition_size(&vdo->layout, + VDO_BLOCK_MAP_PARTITION), + get_partition_size(&vdo->layout, + VDO_RECOVERY_JOURNAL_PARTITION), + get_partition_size(&vdo->layout, + VDO_SLAB_SUMMARY_PARTITION), + &vdo->next_layout); + if (result != VDO_SUCCESS) { + dm_kcopyd_client_destroy(vdo_forget(vdo->partition_copier)); + return result; + } + + /* Ensure the new journal and summary are entirely within the added blocks. */ + min_new_size = (old_size + + get_partition_size(&vdo->next_layout, + VDO_SLAB_SUMMARY_PARTITION) + + get_partition_size(&vdo->next_layout, + VDO_RECOVERY_JOURNAL_PARTITION)); + if (min_new_size > new_size) { + /* Copying the journal and summary would destroy some old metadata. */ + vdo_uninitialize_layout(&vdo->next_layout); + dm_kcopyd_client_destroy(vdo_forget(vdo->partition_copier)); + return VDO_INCREMENT_TOO_SMALL; + } + + return VDO_SUCCESS; +} + +static int prepare_to_grow_physical(struct vdo *vdo, block_count_t new_physical_blocks) +{ + int result; + block_count_t current_physical_blocks = vdo->states.vdo.config.physical_blocks; + + vdo_log_info("Preparing to resize physical to %llu", + (unsigned long long) new_physical_blocks); + VDO_ASSERT_LOG_ONLY((new_physical_blocks > current_physical_blocks), + "New physical size is larger than current physical size"); + result = perform_admin_operation(vdo, PREPARE_GROW_PHYSICAL_PHASE_START, + check_may_grow_physical, + finish_operation_callback, + "prepare grow-physical"); + if (result != VDO_SUCCESS) + return result; + + result = grow_layout(vdo, current_physical_blocks, new_physical_blocks); + if (result != VDO_SUCCESS) + return result; + + result = vdo_prepare_to_grow_slab_depot(vdo->depot, + vdo_get_known_partition(&vdo->next_layout, + VDO_SLAB_DEPOT_PARTITION)); + if (result != VDO_SUCCESS) { + vdo_uninitialize_layout(&vdo->next_layout); + return result; + } + + vdo_log_info("Done preparing to resize physical"); + return VDO_SUCCESS; +} + +/** + * validate_new_device_config() - Check whether a new device config represents a valid modification + * to an existing config. + * @to_validate: The new config to validate. + * @config: The existing config. + * @may_grow: Set to true if growing the logical and physical size of the vdo is currently + * permitted. + * @error_ptr: A pointer to hold the reason for any error. + * + * Return: VDO_SUCCESS or an error. + */ +static int validate_new_device_config(struct device_config *to_validate, + struct device_config *config, bool may_grow, + char **error_ptr) +{ + if (to_validate->owning_target->begin != config->owning_target->begin) { + *error_ptr = "Starting sector cannot change"; + return VDO_PARAMETER_MISMATCH; + } + + if (to_validate->logical_block_size != config->logical_block_size) { + *error_ptr = "Logical block size cannot change"; + return VDO_PARAMETER_MISMATCH; + } + + if (to_validate->logical_blocks < config->logical_blocks) { + *error_ptr = "Can't shrink VDO logical size"; + return VDO_PARAMETER_MISMATCH; + } + + if (to_validate->cache_size != config->cache_size) { + *error_ptr = "Block map cache size cannot change"; + return VDO_PARAMETER_MISMATCH; + } + + if (to_validate->block_map_maximum_age != config->block_map_maximum_age) { + *error_ptr = "Block map maximum age cannot change"; + return VDO_PARAMETER_MISMATCH; + } + + if (memcmp(&to_validate->thread_counts, &config->thread_counts, + sizeof(struct thread_count_config)) != 0) { + *error_ptr = "Thread configuration cannot change"; + return VDO_PARAMETER_MISMATCH; + } + + if (to_validate->physical_blocks < config->physical_blocks) { + *error_ptr = "Removing physical storage from a VDO is not supported"; + return VDO_NOT_IMPLEMENTED; + } + + if (!may_grow && (to_validate->physical_blocks > config->physical_blocks)) { + *error_ptr = "VDO physical size may not grow in current state"; + return VDO_NOT_IMPLEMENTED; + } + + return VDO_SUCCESS; +} + +static int prepare_to_modify(struct dm_target *ti, struct device_config *config, + struct vdo *vdo) +{ + int result; + bool may_grow = (vdo_get_admin_state(vdo) != VDO_ADMIN_STATE_PRE_LOADED); + + result = validate_new_device_config(config, vdo->device_config, may_grow, + &ti->error); + if (result != VDO_SUCCESS) + return -EINVAL; + + if (config->logical_blocks > vdo->device_config->logical_blocks) { + block_count_t logical_blocks = vdo->states.vdo.config.logical_blocks; + + vdo_log_info("Preparing to resize logical to %llu", + (unsigned long long) config->logical_blocks); + VDO_ASSERT_LOG_ONLY((config->logical_blocks > logical_blocks), + "New logical size is larger than current size"); + + result = vdo_prepare_to_grow_block_map(vdo->block_map, + config->logical_blocks); + if (result != VDO_SUCCESS) { + ti->error = "Device vdo_prepare_to_grow_logical failed"; + return result; + } + + vdo_log_info("Done preparing to resize logical"); + } + + if (config->physical_blocks > vdo->device_config->physical_blocks) { + result = prepare_to_grow_physical(vdo, config->physical_blocks); + if (result != VDO_SUCCESS) { + if (result == VDO_PARAMETER_MISMATCH) { + /* + * If we don't trap this case, vdo_status_to_errno() will remap + * it to -EIO, which is misleading and ahistorical. + */ + result = -EINVAL; + } + + if (result == VDO_TOO_MANY_SLABS) + ti->error = "Device vdo_prepare_to_grow_physical failed (specified physical size too big based on formatted slab size)"; + else + ti->error = "Device vdo_prepare_to_grow_physical failed"; + + return result; + } + } + + if (strcmp(config->parent_device_name, vdo->device_config->parent_device_name) != 0) { + const char *device_name = vdo_get_device_name(config->owning_target); + + vdo_log_info("Updating backing device of %s from %s to %s", device_name, + vdo->device_config->parent_device_name, + config->parent_device_name); + } + + return VDO_SUCCESS; +} + +static int update_existing_vdo(const char *device_name, struct dm_target *ti, + unsigned int argc, char **argv, struct vdo *vdo) +{ + int result; + struct device_config *config; + + result = parse_device_config(argc, argv, ti, &config); + if (result != VDO_SUCCESS) + return -EINVAL; + + vdo_log_info("preparing to modify device '%s'", device_name); + result = prepare_to_modify(ti, config, vdo); + if (result != VDO_SUCCESS) { + free_device_config(config); + return vdo_status_to_errno(result); + } + + set_device_config(ti, vdo, config); + return VDO_SUCCESS; +} + +static int vdo_ctr(struct dm_target *ti, unsigned int argc, char **argv) +{ + int result; + struct registered_thread allocating_thread, instance_thread; + const char *device_name; + struct vdo *vdo; + + vdo_register_allocating_thread(&allocating_thread, NULL); + device_name = vdo_get_device_name(ti); + vdo = vdo_find_matching(vdo_is_named, device_name); + if (vdo == NULL) { + result = construct_new_vdo(ti, argc, argv); + } else { + vdo_register_thread_device_id(&instance_thread, &vdo->instance); + result = update_existing_vdo(device_name, ti, argc, argv, vdo); + vdo_unregister_thread_device_id(); + } + + vdo_unregister_allocating_thread(); + return result; +} + +static void vdo_dtr(struct dm_target *ti) +{ + struct device_config *config = ti->private; + struct vdo *vdo = vdo_forget(config->vdo); + + list_del_init(&config->config_list); + if (list_empty(&vdo->device_config_list)) { + const char *device_name; + + /* This was the last config referencing the VDO. Free it. */ + unsigned int instance = vdo->instance; + struct registered_thread allocating_thread, instance_thread; + + vdo_register_thread_device_id(&instance_thread, &instance); + vdo_register_allocating_thread(&allocating_thread, NULL); + + device_name = vdo_get_device_name(ti); + vdo_log_info("stopping device '%s'", device_name); + if (vdo->dump_on_shutdown) + vdo_dump_all(vdo, "device shutdown"); + + vdo_destroy(vdo_forget(vdo)); + vdo_log_info("device '%s' stopped", device_name); + vdo_unregister_thread_device_id(); + vdo_unregister_allocating_thread(); + release_instance(instance); + } else if (config == vdo->device_config) { + /* + * The VDO still references this config. Give it a reference to a config that isn't + * being destroyed. + */ + vdo->device_config = list_first_entry(&vdo->device_config_list, + struct device_config, config_list); + } + + free_device_config(config); + ti->private = NULL; +} + +static void vdo_presuspend(struct dm_target *ti) +{ + get_vdo_for_target(ti)->suspend_type = + (dm_noflush_suspending(ti) ? VDO_ADMIN_STATE_SUSPENDING : VDO_ADMIN_STATE_SAVING); +} + +/** + * write_super_block_for_suspend() - Update the VDO state and save the super block. + * @completion: The admin completion + */ +static void write_super_block_for_suspend(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + + switch (vdo_get_state(vdo)) { + case VDO_DIRTY: + case VDO_NEW: + vdo_set_state(vdo, VDO_CLEAN); + break; + + case VDO_CLEAN: + case VDO_READ_ONLY_MODE: + case VDO_FORCE_REBUILD: + case VDO_RECOVERING: + case VDO_REBUILD_FOR_UPGRADE: + break; + + case VDO_REPLAYING: + default: + vdo_continue_completion(completion, UDS_BAD_STATE); + return; + } + + vdo_save_components(vdo, completion); +} + +/** + * suspend_callback() - Callback to initiate a suspend, registered in vdo_postsuspend(). + * @completion: The sub-task completion. + */ +static void suspend_callback(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + struct admin_state *state = &vdo->admin.state; + int result; + + assert_admin_phase_thread(vdo, __func__); + + switch (advance_phase(vdo)) { + case SUSPEND_PHASE_START: + if (vdo_get_admin_state_code(state)->quiescent) { + /* Already suspended */ + break; + } + + vdo_continue_completion(completion, + vdo_start_operation(state, vdo->suspend_type)); + return; + + case SUSPEND_PHASE_PACKER: + /* + * If the VDO was already resumed from a prior suspend while read-only, some of the + * components may not have been resumed. By setting a read-only error here, we + * guarantee that the result of this suspend will be VDO_READ_ONLY and not + * VDO_INVALID_ADMIN_STATE in that case. + */ + if (vdo_in_read_only_mode(vdo)) + vdo_set_completion_result(completion, VDO_READ_ONLY); + + vdo_drain_packer(vdo->packer, completion); + return; + + case SUSPEND_PHASE_DATA_VIOS: + drain_data_vio_pool(vdo->data_vio_pool, completion); + return; + + case SUSPEND_PHASE_DEDUPE: + vdo_drain_hash_zones(vdo->hash_zones, completion); + return; + + case SUSPEND_PHASE_FLUSHES: + vdo_drain_flusher(vdo->flusher, completion); + return; + + case SUSPEND_PHASE_LOGICAL_ZONES: + /* + * Attempt to flush all I/O before completing post suspend work. We believe a + * suspended device is expected to have persisted all data written before the + * suspend, even if it hasn't been flushed yet. + */ + result = vdo_synchronous_flush(vdo); + if (result != VDO_SUCCESS) + vdo_enter_read_only_mode(vdo, result); + + vdo_drain_logical_zones(vdo->logical_zones, + vdo_get_admin_state_code(state), completion); + return; + + case SUSPEND_PHASE_BLOCK_MAP: + vdo_drain_block_map(vdo->block_map, vdo_get_admin_state_code(state), + completion); + return; + + case SUSPEND_PHASE_JOURNAL: + vdo_drain_recovery_journal(vdo->recovery_journal, + vdo_get_admin_state_code(state), completion); + return; + + case SUSPEND_PHASE_DEPOT: + vdo_drain_slab_depot(vdo->depot, vdo_get_admin_state_code(state), + completion); + return; + + case SUSPEND_PHASE_READ_ONLY_WAIT: + vdo_wait_until_not_entering_read_only_mode(completion); + return; + + case SUSPEND_PHASE_WRITE_SUPER_BLOCK: + if (vdo_is_state_suspending(state) || (completion->result != VDO_SUCCESS)) { + /* If we didn't save the VDO or there was an error, we're done. */ + break; + } + + write_super_block_for_suspend(completion); + return; + + case SUSPEND_PHASE_END: + break; + + default: + vdo_set_completion_result(completion, UDS_BAD_STATE); + } + + finish_operation_callback(completion); +} + +static void vdo_postsuspend(struct dm_target *ti) +{ + struct vdo *vdo = get_vdo_for_target(ti); + struct registered_thread instance_thread; + const char *device_name; + int result; + + vdo_register_thread_device_id(&instance_thread, &vdo->instance); + device_name = vdo_get_device_name(vdo->device_config->owning_target); + vdo_log_info("suspending device '%s'", device_name); + + /* + * It's important to note any error here does not actually stop device-mapper from + * suspending the device. All this work is done post suspend. + */ + result = perform_admin_operation(vdo, SUSPEND_PHASE_START, suspend_callback, + suspend_callback, "suspend"); + + if ((result == VDO_SUCCESS) || (result == VDO_READ_ONLY)) { + /* + * Treat VDO_READ_ONLY as a success since a read-only suspension still leaves the + * VDO suspended. + */ + vdo_log_info("device '%s' suspended", device_name); + } else if (result == VDO_INVALID_ADMIN_STATE) { + vdo_log_error("Suspend invoked while in unexpected state: %s", + vdo_get_admin_state(vdo)->name); + } else { + vdo_log_error_strerror(result, "Suspend of device '%s' failed", + device_name); + } + + vdo_unregister_thread_device_id(); +} + +/** + * was_new() - Check whether the vdo was new when it was loaded. + * @vdo: The vdo to query. + * + * Return: true if the vdo was new. + */ +static bool was_new(const struct vdo *vdo) +{ + return (vdo->load_state == VDO_NEW); +} + +/** + * requires_repair() - Check whether a vdo requires recovery or rebuild. + * @vdo: The vdo to query. + * + * Return: true if the vdo must be repaired. + */ +static bool __must_check requires_repair(const struct vdo *vdo) +{ + switch (vdo_get_state(vdo)) { + case VDO_DIRTY: + case VDO_FORCE_REBUILD: + case VDO_REPLAYING: + case VDO_REBUILD_FOR_UPGRADE: + return true; + + default: + return false; + } +} + +/** + * get_load_type() - Determine how the slab depot was loaded. + * @vdo: The vdo. + * + * Return: How the depot was loaded. + */ +static enum slab_depot_load_type get_load_type(struct vdo *vdo) +{ + if (vdo_state_requires_read_only_rebuild(vdo->load_state)) + return VDO_SLAB_DEPOT_REBUILD_LOAD; + + if (vdo_state_requires_recovery(vdo->load_state)) + return VDO_SLAB_DEPOT_RECOVERY_LOAD; + + return VDO_SLAB_DEPOT_NORMAL_LOAD; +} + +/** + * load_callback() - Callback to do the destructive parts of loading a VDO. + * @completion: The sub-task completion. + */ +static void load_callback(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + int result; + + assert_admin_phase_thread(vdo, __func__); + + switch (advance_phase(vdo)) { + case LOAD_PHASE_START: + result = vdo_start_operation(&vdo->admin.state, VDO_ADMIN_STATE_LOADING); + if (result != VDO_SUCCESS) { + vdo_continue_completion(completion, result); + return; + } + + /* Prepare the recovery journal for new entries. */ + vdo_open_recovery_journal(vdo->recovery_journal, vdo->depot, + vdo->block_map); + vdo_allow_read_only_mode_entry(completion); + return; + + case LOAD_PHASE_LOAD_DEPOT: + vdo_set_dedupe_state_normal(vdo->hash_zones); + if (vdo_is_read_only(vdo)) { + /* + * In read-only mode we don't use the allocator and it may not even be + * readable, so don't bother trying to load it. + */ + vdo_set_completion_result(completion, VDO_READ_ONLY); + break; + } + + if (requires_repair(vdo)) { + vdo_repair(completion); + return; + } + + vdo_load_slab_depot(vdo->depot, + (was_new(vdo) ? VDO_ADMIN_STATE_FORMATTING : + VDO_ADMIN_STATE_LOADING), + completion, NULL); + return; + + case LOAD_PHASE_MAKE_DIRTY: + vdo_set_state(vdo, VDO_DIRTY); + vdo_save_components(vdo, completion); + return; + + case LOAD_PHASE_PREPARE_TO_ALLOCATE: + vdo_initialize_block_map_from_journal(vdo->block_map, + vdo->recovery_journal); + vdo_prepare_slab_depot_to_allocate(vdo->depot, get_load_type(vdo), + completion); + return; + + case LOAD_PHASE_SCRUB_SLABS: + if (vdo_state_requires_recovery(vdo->load_state)) + vdo_enter_recovery_mode(vdo); + + vdo_scrub_all_unrecovered_slabs(vdo->depot, completion); + return; + + case LOAD_PHASE_DATA_REDUCTION: + WRITE_ONCE(vdo->compressing, vdo->device_config->compression); + if (vdo->device_config->deduplication) { + /* + * Don't try to load or rebuild the index first (and log scary error + * messages) if this is known to be a newly-formatted volume. + */ + vdo_start_dedupe_index(vdo->hash_zones, was_new(vdo)); + } + + vdo->allocations_allowed = false; + fallthrough; + + case LOAD_PHASE_FINISHED: + break; + + case LOAD_PHASE_DRAIN_JOURNAL: + vdo_drain_recovery_journal(vdo->recovery_journal, VDO_ADMIN_STATE_SAVING, + completion); + return; + + case LOAD_PHASE_WAIT_FOR_READ_ONLY: + /* Avoid an infinite loop */ + completion->error_handler = NULL; + vdo->admin.phase = LOAD_PHASE_FINISHED; + vdo_wait_until_not_entering_read_only_mode(completion); + return; + + default: + vdo_set_completion_result(completion, UDS_BAD_STATE); + } + + finish_operation_callback(completion); +} + +/** + * handle_load_error() - Handle an error during the load operation. + * @completion: The admin completion. + * + * If at all possible, brings the vdo online in read-only mode. This handler is registered in + * vdo_preresume_registered(). + */ +static void handle_load_error(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + + if (vdo_requeue_completion_if_needed(completion, + vdo->thread_config.admin_thread)) + return; + + if (vdo_state_requires_read_only_rebuild(vdo->load_state) && + (vdo->admin.phase == LOAD_PHASE_MAKE_DIRTY)) { + vdo_log_error_strerror(completion->result, "aborting load"); + vdo->admin.phase = LOAD_PHASE_DRAIN_JOURNAL; + load_callback(vdo_forget(completion)); + return; + } + + vdo_log_error_strerror(completion->result, + "Entering read-only mode due to load error"); + vdo->admin.phase = LOAD_PHASE_WAIT_FOR_READ_ONLY; + vdo_enter_read_only_mode(vdo, completion->result); + completion->result = VDO_READ_ONLY; + load_callback(completion); +} + +/** + * write_super_block_for_resume() - Update the VDO state and save the super block. + * @completion: The admin completion + */ +static void write_super_block_for_resume(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + + switch (vdo_get_state(vdo)) { + case VDO_CLEAN: + case VDO_NEW: + vdo_set_state(vdo, VDO_DIRTY); + vdo_save_components(vdo, completion); + return; + + case VDO_DIRTY: + case VDO_READ_ONLY_MODE: + case VDO_FORCE_REBUILD: + case VDO_RECOVERING: + case VDO_REBUILD_FOR_UPGRADE: + /* No need to write the super block in these cases */ + vdo_launch_completion(completion); + return; + + case VDO_REPLAYING: + default: + vdo_continue_completion(completion, UDS_BAD_STATE); + } +} + +/** + * resume_callback() - Callback to resume a VDO. + * @completion: The admin completion. + */ +static void resume_callback(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + int result; + + assert_admin_phase_thread(vdo, __func__); + + switch (advance_phase(vdo)) { + case RESUME_PHASE_START: + result = vdo_start_operation(&vdo->admin.state, + VDO_ADMIN_STATE_RESUMING); + if (result != VDO_SUCCESS) { + vdo_continue_completion(completion, result); + return; + } + + write_super_block_for_resume(completion); + return; + + case RESUME_PHASE_ALLOW_READ_ONLY_MODE: + vdo_allow_read_only_mode_entry(completion); + return; + + case RESUME_PHASE_DEDUPE: + vdo_resume_hash_zones(vdo->hash_zones, completion); + return; + + case RESUME_PHASE_DEPOT: + vdo_resume_slab_depot(vdo->depot, completion); + return; + + case RESUME_PHASE_JOURNAL: + vdo_resume_recovery_journal(vdo->recovery_journal, completion); + return; + + case RESUME_PHASE_BLOCK_MAP: + vdo_resume_block_map(vdo->block_map, completion); + return; + + case RESUME_PHASE_LOGICAL_ZONES: + vdo_resume_logical_zones(vdo->logical_zones, completion); + return; + + case RESUME_PHASE_PACKER: + { + bool was_enabled = vdo_get_compressing(vdo); + bool enable = vdo->device_config->compression; + + if (enable != was_enabled) + WRITE_ONCE(vdo->compressing, enable); + vdo_log_info("compression is %s", (enable ? "enabled" : "disabled")); + + vdo_resume_packer(vdo->packer, completion); + return; + } + + case RESUME_PHASE_FLUSHER: + vdo_resume_flusher(vdo->flusher, completion); + return; + + case RESUME_PHASE_DATA_VIOS: + resume_data_vio_pool(vdo->data_vio_pool, completion); + return; + + case RESUME_PHASE_END: + break; + + default: + vdo_set_completion_result(completion, UDS_BAD_STATE); + } + + finish_operation_callback(completion); +} + +/** + * grow_logical_callback() - Callback to initiate a grow logical. + * @completion: The admin completion. + * + * Registered in perform_grow_logical(). + */ +static void grow_logical_callback(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + int result; + + assert_admin_phase_thread(vdo, __func__); + + switch (advance_phase(vdo)) { + case GROW_LOGICAL_PHASE_START: + if (vdo_is_read_only(vdo)) { + vdo_log_error_strerror(VDO_READ_ONLY, + "Can't grow logical size of a read-only VDO"); + vdo_set_completion_result(completion, VDO_READ_ONLY); + break; + } + + result = vdo_start_operation(&vdo->admin.state, + VDO_ADMIN_STATE_SUSPENDED_OPERATION); + if (result != VDO_SUCCESS) { + vdo_continue_completion(completion, result); + return; + } + + vdo->states.vdo.config.logical_blocks = vdo->block_map->next_entry_count; + vdo_save_components(vdo, completion); + return; + + case GROW_LOGICAL_PHASE_GROW_BLOCK_MAP: + vdo_grow_block_map(vdo->block_map, completion); + return; + + case GROW_LOGICAL_PHASE_END: + break; + + case GROW_LOGICAL_PHASE_ERROR: + vdo_enter_read_only_mode(vdo, completion->result); + break; + + default: + vdo_set_completion_result(completion, UDS_BAD_STATE); + } + + finish_operation_callback(completion); +} + +/** + * handle_logical_growth_error() - Handle an error during the grow physical process. + * @completion: The admin completion. + */ +static void handle_logical_growth_error(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + + if (vdo->admin.phase == GROW_LOGICAL_PHASE_GROW_BLOCK_MAP) { + /* + * We've failed to write the new size in the super block, so set our in memory + * config back to the old size. + */ + vdo->states.vdo.config.logical_blocks = vdo->block_map->entry_count; + vdo_abandon_block_map_growth(vdo->block_map); + } + + vdo->admin.phase = GROW_LOGICAL_PHASE_ERROR; + grow_logical_callback(completion); +} + +/** + * perform_grow_logical() - Grow the logical size of the vdo. + * @vdo: The vdo to grow. + * @new_logical_blocks: The size to which the vdo should be grown. + * + * Context: This method may only be called when the vdo has been suspended and must not be called + * from a base thread. + * + * Return: VDO_SUCCESS or an error. + */ +static int perform_grow_logical(struct vdo *vdo, block_count_t new_logical_blocks) +{ + int result; + + if (vdo->device_config->logical_blocks == new_logical_blocks) { + /* + * A table was loaded for which we prepared to grow, but a table without that + * growth was what we are resuming with. + */ + vdo_abandon_block_map_growth(vdo->block_map); + return VDO_SUCCESS; + } + + vdo_log_info("Resizing logical to %llu", + (unsigned long long) new_logical_blocks); + if (vdo->block_map->next_entry_count != new_logical_blocks) + return VDO_PARAMETER_MISMATCH; + + result = perform_admin_operation(vdo, GROW_LOGICAL_PHASE_START, + grow_logical_callback, + handle_logical_growth_error, "grow logical"); + if (result != VDO_SUCCESS) + return result; + + vdo_log_info("Logical blocks now %llu", (unsigned long long) new_logical_blocks); + return VDO_SUCCESS; +} + +static void copy_callback(int read_err, unsigned long write_err, void *context) +{ + struct vdo_completion *completion = context; + int result = (((read_err == 0) && (write_err == 0)) ? VDO_SUCCESS : -EIO); + + vdo_continue_completion(completion, result); +} + +static void partition_to_region(struct partition *partition, struct vdo *vdo, + struct dm_io_region *region) +{ + physical_block_number_t pbn = partition->offset - vdo->geometry.bio_offset; + + *region = (struct dm_io_region) { + .bdev = vdo_get_backing_device(vdo), + .sector = pbn * VDO_SECTORS_PER_BLOCK, + .count = partition->count * VDO_SECTORS_PER_BLOCK, + }; +} + +/** + * copy_partition() - Copy a partition from the location specified in the current layout to that in + * the next layout. + * @vdo: The vdo preparing to grow. + * @id: The ID of the partition to copy. + * @parent: The completion to notify when the copy is complete. + */ +static void copy_partition(struct vdo *vdo, enum partition_id id, + struct vdo_completion *parent) +{ + struct dm_io_region read_region, write_regions[1]; + struct partition *from = vdo_get_known_partition(&vdo->layout, id); + struct partition *to = vdo_get_known_partition(&vdo->next_layout, id); + + partition_to_region(from, vdo, &read_region); + partition_to_region(to, vdo, &write_regions[0]); + dm_kcopyd_copy(vdo->partition_copier, &read_region, 1, write_regions, 0, + copy_callback, parent); +} + +/** + * grow_physical_callback() - Callback to initiate a grow physical. + * @completion: The admin completion. + * + * Registered in perform_grow_physical(). + */ +static void grow_physical_callback(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + int result; + + assert_admin_phase_thread(vdo, __func__); + + switch (advance_phase(vdo)) { + case GROW_PHYSICAL_PHASE_START: + if (vdo_is_read_only(vdo)) { + vdo_log_error_strerror(VDO_READ_ONLY, + "Can't grow physical size of a read-only VDO"); + vdo_set_completion_result(completion, VDO_READ_ONLY); + break; + } + + result = vdo_start_operation(&vdo->admin.state, + VDO_ADMIN_STATE_SUSPENDED_OPERATION); + if (result != VDO_SUCCESS) { + vdo_continue_completion(completion, result); + return; + } + + /* Copy the journal into the new layout. */ + copy_partition(vdo, VDO_RECOVERY_JOURNAL_PARTITION, completion); + return; + + case GROW_PHYSICAL_PHASE_COPY_SUMMARY: + copy_partition(vdo, VDO_SLAB_SUMMARY_PARTITION, completion); + return; + + case GROW_PHYSICAL_PHASE_UPDATE_COMPONENTS: + vdo_uninitialize_layout(&vdo->layout); + vdo->layout = vdo->next_layout; + vdo_forget(vdo->next_layout.head); + vdo->states.vdo.config.physical_blocks = vdo->layout.size; + vdo_update_slab_depot_size(vdo->depot); + vdo_save_components(vdo, completion); + return; + + case GROW_PHYSICAL_PHASE_USE_NEW_SLABS: + vdo_use_new_slabs(vdo->depot, completion); + return; + + case GROW_PHYSICAL_PHASE_END: + vdo->depot->summary_origin = + vdo_get_known_partition(&vdo->layout, + VDO_SLAB_SUMMARY_PARTITION)->offset; + vdo->recovery_journal->origin = + vdo_get_known_partition(&vdo->layout, + VDO_RECOVERY_JOURNAL_PARTITION)->offset; + break; + + case GROW_PHYSICAL_PHASE_ERROR: + vdo_enter_read_only_mode(vdo, completion->result); + break; + + default: + vdo_set_completion_result(completion, UDS_BAD_STATE); + } + + vdo_uninitialize_layout(&vdo->next_layout); + finish_operation_callback(completion); +} + +/** + * handle_physical_growth_error() - Handle an error during the grow physical process. + * @completion: The sub-task completion. + */ +static void handle_physical_growth_error(struct vdo_completion *completion) +{ + completion->vdo->admin.phase = GROW_PHYSICAL_PHASE_ERROR; + grow_physical_callback(completion); +} + +/** + * perform_grow_physical() - Grow the physical size of the vdo. + * @vdo: The vdo to resize. + * @new_physical_blocks: The new physical size in blocks. + * + * Context: This method may only be called when the vdo has been suspended and must not be called + * from a base thread. + * + * Return: VDO_SUCCESS or an error. + */ +static int perform_grow_physical(struct vdo *vdo, block_count_t new_physical_blocks) +{ + int result; + block_count_t new_depot_size, prepared_depot_size; + block_count_t old_physical_blocks = vdo->states.vdo.config.physical_blocks; + + /* Skip any noop grows. */ + if (old_physical_blocks == new_physical_blocks) + return VDO_SUCCESS; + + if (new_physical_blocks != vdo->next_layout.size) { + /* + * Either the VDO isn't prepared to grow, or it was prepared to grow to a different + * size. Doing this check here relies on the fact that the call to this method is + * done under the dmsetup message lock. + */ + vdo_uninitialize_layout(&vdo->next_layout); + vdo_abandon_new_slabs(vdo->depot); + return VDO_PARAMETER_MISMATCH; + } + + /* Validate that we are prepared to grow appropriately. */ + new_depot_size = + vdo_get_known_partition(&vdo->next_layout, VDO_SLAB_DEPOT_PARTITION)->count; + prepared_depot_size = (vdo->depot->new_slabs == NULL) ? 0 : vdo->depot->new_size; + if (prepared_depot_size != new_depot_size) + return VDO_PARAMETER_MISMATCH; + + result = perform_admin_operation(vdo, GROW_PHYSICAL_PHASE_START, + grow_physical_callback, + handle_physical_growth_error, "grow physical"); + if (result != VDO_SUCCESS) + return result; + + vdo_log_info("Physical block count was %llu, now %llu", + (unsigned long long) old_physical_blocks, + (unsigned long long) new_physical_blocks); + return VDO_SUCCESS; +} + +/** + * apply_new_vdo_configuration() - Attempt to make any configuration changes from the table being + * resumed. + * @vdo: The vdo being resumed. + * @config: The new device configuration derived from the table with which the vdo is being + * resumed. + * + * Return: VDO_SUCCESS or an error. + */ +static int __must_check apply_new_vdo_configuration(struct vdo *vdo, + struct device_config *config) +{ + int result; + + result = perform_grow_logical(vdo, config->logical_blocks); + if (result != VDO_SUCCESS) { + vdo_log_error("grow logical operation failed, result = %d", result); + return result; + } + + result = perform_grow_physical(vdo, config->physical_blocks); + if (result != VDO_SUCCESS) + vdo_log_error("resize operation failed, result = %d", result); + + return result; +} + +static int vdo_preresume_registered(struct dm_target *ti, struct vdo *vdo) +{ + struct device_config *config = ti->private; + const char *device_name = vdo_get_device_name(ti); + block_count_t backing_blocks; + int result; + + backing_blocks = get_underlying_device_block_count(vdo); + if (backing_blocks < config->physical_blocks) { + /* FIXME: can this still happen? */ + vdo_log_error("resume of device '%s' failed: backing device has %llu blocks but VDO physical size is %llu blocks", + device_name, (unsigned long long) backing_blocks, + (unsigned long long) config->physical_blocks); + return -EINVAL; + } + + if (vdo_get_admin_state(vdo) == VDO_ADMIN_STATE_PRE_LOADED) { + vdo_log_info("starting device '%s'", device_name); + result = perform_admin_operation(vdo, LOAD_PHASE_START, load_callback, + handle_load_error, "load"); + if ((result != VDO_SUCCESS) && (result != VDO_READ_ONLY)) { + /* + * Something has gone very wrong. Make sure everything has drained and + * leave the device in an unresumable state. + */ + vdo_log_error_strerror(result, + "Start failed, could not load VDO metadata"); + vdo->suspend_type = VDO_ADMIN_STATE_STOPPING; + perform_admin_operation(vdo, SUSPEND_PHASE_START, + suspend_callback, suspend_callback, + "suspend"); + return result; + } + + /* Even if the VDO is read-only, it is now able to handle read requests. */ + vdo_log_info("device '%s' started", device_name); + } + + vdo_log_info("resuming device '%s'", device_name); + + /* If this fails, the VDO was not in a state to be resumed. This should never happen. */ + result = apply_new_vdo_configuration(vdo, config); + BUG_ON(result == VDO_INVALID_ADMIN_STATE); + + /* + * Now that we've tried to modify the vdo, the new config *is* the config, whether the + * modifications worked or not. + */ + vdo->device_config = config; + + /* + * Any error here is highly unexpected and the state of the vdo is questionable, so we mark + * it read-only in memory. Because we are suspended, the read-only state will not be + * written to disk. + */ + if (result != VDO_SUCCESS) { + vdo_log_error_strerror(result, + "Commit of modifications to device '%s' failed", + device_name); + vdo_enter_read_only_mode(vdo, result); + return result; + } + + if (vdo_get_admin_state(vdo)->normal) { + /* The VDO was just started, so we don't need to resume it. */ + return VDO_SUCCESS; + } + + result = perform_admin_operation(vdo, RESUME_PHASE_START, resume_callback, + resume_callback, "resume"); + BUG_ON(result == VDO_INVALID_ADMIN_STATE); + if (result == VDO_READ_ONLY) { + /* Even if the vdo is read-only, it has still resumed. */ + result = VDO_SUCCESS; + } + + if (result != VDO_SUCCESS) + vdo_log_error("resume of device '%s' failed with error: %d", device_name, + result); + + return result; +} + +static int vdo_preresume(struct dm_target *ti) +{ + struct registered_thread instance_thread; + struct vdo *vdo = get_vdo_for_target(ti); + int result; + + vdo_register_thread_device_id(&instance_thread, &vdo->instance); + result = vdo_preresume_registered(ti, vdo); + if ((result == VDO_PARAMETER_MISMATCH) || (result == VDO_INVALID_ADMIN_STATE)) + result = -EINVAL; + vdo_unregister_thread_device_id(); + return vdo_status_to_errno(result); +} + +static void vdo_resume(struct dm_target *ti) +{ + struct registered_thread instance_thread; + + vdo_register_thread_device_id(&instance_thread, + &get_vdo_for_target(ti)->instance); + vdo_log_info("device '%s' resumed", vdo_get_device_name(ti)); + vdo_unregister_thread_device_id(); +} + +/* + * If anything changes that affects how user tools will interact with vdo, update the version + * number and make sure documentation about the change is complete so tools can properly update + * their management code. + */ +static struct target_type vdo_target_bio = { + .features = DM_TARGET_SINGLETON, + .name = "vdo", + .version = { 9, 0, 0 }, + .module = THIS_MODULE, + .ctr = vdo_ctr, + .dtr = vdo_dtr, + .io_hints = vdo_io_hints, + .iterate_devices = vdo_iterate_devices, + .map = vdo_map_bio, + .message = vdo_message, + .status = vdo_status, + .presuspend = vdo_presuspend, + .postsuspend = vdo_postsuspend, + .preresume = vdo_preresume, + .resume = vdo_resume, +}; + +static bool dm_registered; + +static void vdo_module_destroy(void) +{ + vdo_log_debug("unloading"); + + if (dm_registered) + dm_unregister_target(&vdo_target_bio); + + VDO_ASSERT_LOG_ONLY(instances.count == 0, + "should have no instance numbers still in use, but have %u", + instances.count); + vdo_free(instances.words); + memset(&instances, 0, sizeof(struct instance_tracker)); +} + +static int __init vdo_init(void) +{ + int result = 0; + + /* Memory tracking must be initialized first for accurate accounting. */ + vdo_memory_init(); + vdo_initialize_threads_mutex(); + vdo_initialize_thread_device_registry(); + vdo_initialize_device_registry_once(); + + /* Add VDO errors to the set of errors registered by the indexer. */ + result = vdo_register_status_codes(); + if (result != VDO_SUCCESS) { + vdo_log_error("vdo_register_status_codes failed %d", result); + vdo_module_destroy(); + return result; + } + + result = dm_register_target(&vdo_target_bio); + if (result < 0) { + vdo_log_error("dm_register_target failed %d", result); + vdo_module_destroy(); + return result; + } + dm_registered = true; + + return result; +} + +static void __exit vdo_exit(void) +{ + vdo_module_destroy(); + /* Memory tracking cleanup must be done last. */ + vdo_memory_exit(); +} + +module_init(vdo_init); +module_exit(vdo_exit); + +module_param_named(log_level, vdo_log_level, uint, 0644); +MODULE_PARM_DESC(log_level, "Log level for log messages"); + +MODULE_DESCRIPTION(DM_NAME " target for transparent deduplication"); +MODULE_AUTHOR("Red Hat, Inc."); +MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm-vdo/dump.c b/drivers/md/dm-vdo/dump.c new file mode 100644 index 000000000000..00e575d7d773 --- /dev/null +++ b/drivers/md/dm-vdo/dump.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "dump.h" + +#include + +#include "memory-alloc.h" +#include "string-utils.h" + +#include "constants.h" +#include "data-vio.h" +#include "dedupe.h" +#include "funnel-workqueue.h" +#include "io-submitter.h" +#include "logger.h" +#include "types.h" +#include "vdo.h" + +enum dump_options { + /* Work queues */ + SHOW_QUEUES, + /* Memory pools */ + SHOW_VIO_POOL, + /* Others */ + SHOW_VDO_STATUS, + /* This one means an option overrides the "default" choices, instead of altering them. */ + SKIP_DEFAULT +}; + +enum dump_option_flags { + /* Work queues */ + FLAG_SHOW_QUEUES = (1 << SHOW_QUEUES), + /* Memory pools */ + FLAG_SHOW_VIO_POOL = (1 << SHOW_VIO_POOL), + /* Others */ + FLAG_SHOW_VDO_STATUS = (1 << SHOW_VDO_STATUS), + /* Special */ + FLAG_SKIP_DEFAULT = (1 << SKIP_DEFAULT) +}; + +#define FLAGS_ALL_POOLS (FLAG_SHOW_VIO_POOL) +#define DEFAULT_DUMP_FLAGS (FLAG_SHOW_QUEUES | FLAG_SHOW_VDO_STATUS) +/* Another static buffer... log10(256) = 2.408+, round up: */ +#define DIGITS_PER_U64 (1 + sizeof(u64) * 2409 / 1000) + +static inline bool is_arg_string(const char *arg, const char *this_option) +{ + /* convention seems to be case-independent options */ + return strncasecmp(arg, this_option, strlen(this_option)) == 0; +} + +static void do_dump(struct vdo *vdo, unsigned int dump_options_requested, + const char *why) +{ + u32 active, maximum; + s64 outstanding; + + vdo_log_info("%s dump triggered via %s", VDO_LOGGING_MODULE_NAME, why); + active = get_data_vio_pool_active_requests(vdo->data_vio_pool); + maximum = get_data_vio_pool_maximum_requests(vdo->data_vio_pool); + outstanding = (atomic64_read(&vdo->stats.bios_submitted) - + atomic64_read(&vdo->stats.bios_completed)); + vdo_log_info("%u device requests outstanding (max %u), %lld bio requests outstanding, device '%s'", + active, maximum, outstanding, + vdo_get_device_name(vdo->device_config->owning_target)); + if (((dump_options_requested & FLAG_SHOW_QUEUES) != 0) && (vdo->threads != NULL)) { + thread_id_t id; + + for (id = 0; id < vdo->thread_config.thread_count; id++) + vdo_dump_work_queue(vdo->threads[id].queue); + } + + vdo_dump_hash_zones(vdo->hash_zones); + dump_data_vio_pool(vdo->data_vio_pool, + (dump_options_requested & FLAG_SHOW_VIO_POOL) != 0); + if ((dump_options_requested & FLAG_SHOW_VDO_STATUS) != 0) + vdo_dump_status(vdo); + + vdo_report_memory_usage(); + vdo_log_info("end of %s dump", VDO_LOGGING_MODULE_NAME); +} + +static int parse_dump_options(unsigned int argc, char *const *argv, + unsigned int *dump_options_requested_ptr) +{ + unsigned int dump_options_requested = 0; + + static const struct { + const char *name; + unsigned int flags; + } option_names[] = { + { "viopool", FLAG_SKIP_DEFAULT | FLAG_SHOW_VIO_POOL }, + { "vdo", FLAG_SKIP_DEFAULT | FLAG_SHOW_VDO_STATUS }, + { "pools", FLAG_SKIP_DEFAULT | FLAGS_ALL_POOLS }, + { "queues", FLAG_SKIP_DEFAULT | FLAG_SHOW_QUEUES }, + { "threads", FLAG_SKIP_DEFAULT | FLAG_SHOW_QUEUES }, + { "default", FLAG_SKIP_DEFAULT | DEFAULT_DUMP_FLAGS }, + { "all", ~0 }, + }; + + bool options_okay = true; + unsigned int i; + + for (i = 1; i < argc; i++) { + unsigned int j; + + for (j = 0; j < ARRAY_SIZE(option_names); j++) { + if (is_arg_string(argv[i], option_names[j].name)) { + dump_options_requested |= option_names[j].flags; + break; + } + } + if (j == ARRAY_SIZE(option_names)) { + vdo_log_warning("dump option name '%s' unknown", argv[i]); + options_okay = false; + } + } + if (!options_okay) + return -EINVAL; + if ((dump_options_requested & FLAG_SKIP_DEFAULT) == 0) + dump_options_requested |= DEFAULT_DUMP_FLAGS; + *dump_options_requested_ptr = dump_options_requested; + return 0; +} + +/* Dump as specified by zero or more string arguments. */ +int vdo_dump(struct vdo *vdo, unsigned int argc, char *const *argv, const char *why) +{ + unsigned int dump_options_requested = 0; + int result = parse_dump_options(argc, argv, &dump_options_requested); + + if (result != 0) + return result; + + do_dump(vdo, dump_options_requested, why); + return 0; +} + +/* Dump everything we know how to dump */ +void vdo_dump_all(struct vdo *vdo, const char *why) +{ + do_dump(vdo, ~0, why); +} + +/* + * Dump out the data_vio waiters on a waitq. + * wait_on should be the label to print for queue (e.g. logical or physical) + */ +static void dump_vio_waiters(struct vdo_wait_queue *waitq, char *wait_on) +{ + struct vdo_waiter *waiter, *first = vdo_waitq_get_first_waiter(waitq); + struct data_vio *data_vio; + + if (first == NULL) + return; + + data_vio = vdo_waiter_as_data_vio(first); + + vdo_log_info(" %s is locked. Waited on by: vio %px pbn %llu lbn %llu d-pbn %llu lastOp %s", + wait_on, data_vio, data_vio->allocation.pbn, data_vio->logical.lbn, + data_vio->duplicate.pbn, get_data_vio_operation_name(data_vio)); + + for (waiter = first->next_waiter; waiter != first; waiter = waiter->next_waiter) { + data_vio = vdo_waiter_as_data_vio(waiter); + vdo_log_info(" ... and : vio %px pbn %llu lbn %llu d-pbn %llu lastOp %s", + data_vio, data_vio->allocation.pbn, data_vio->logical.lbn, + data_vio->duplicate.pbn, + get_data_vio_operation_name(data_vio)); + } +} + +/* + * Encode various attributes of a data_vio as a string of one-character flags. This encoding is for + * logging brevity: + * + * R => vio completion result not VDO_SUCCESS + * W => vio is on a waitq + * D => vio is a duplicate + * p => vio is a partial block operation + * z => vio is a zero block + * d => vio is a discard + * + * The common case of no flags set will result in an empty, null-terminated buffer. If any flags + * are encoded, the first character in the string will be a space character. + */ +static void encode_vio_dump_flags(struct data_vio *data_vio, char buffer[8]) +{ + char *p_flag = buffer; + *p_flag++ = ' '; + if (data_vio->vio.completion.result != VDO_SUCCESS) + *p_flag++ = 'R'; + if (data_vio->waiter.next_waiter != NULL) + *p_flag++ = 'W'; + if (data_vio->is_duplicate) + *p_flag++ = 'D'; + if (data_vio->is_partial) + *p_flag++ = 'p'; + if (data_vio->is_zero) + *p_flag++ = 'z'; + if (data_vio->remaining_discard > 0) + *p_flag++ = 'd'; + if (p_flag == &buffer[1]) { + /* No flags, so remove the blank space. */ + p_flag = buffer; + } + *p_flag = '\0'; +} + +/* Implements buffer_dump_function. */ +void dump_data_vio(void *data) +{ + struct data_vio *data_vio = data; + + /* + * This just needs to be big enough to hold a queue (thread) name and a function name (plus + * a separator character and NUL). The latter is limited only by taste. + * + * In making this static, we're assuming only one "dump" will run at a time. If more than + * one does run, the log output will be garbled anyway. + */ + static char vio_completion_dump_buffer[100 + MAX_VDO_WORK_QUEUE_NAME_LEN]; + static char vio_block_number_dump_buffer[sizeof("P L D") + 3 * DIGITS_PER_U64]; + static char vio_flush_generation_buffer[sizeof(" FG") + DIGITS_PER_U64]; + static char flags_dump_buffer[8]; + + /* + * We're likely to be logging a couple thousand of these lines, and in some circumstances + * syslogd may have trouble keeping up, so keep it BRIEF rather than user-friendly. + */ + vdo_dump_completion_to_buffer(&data_vio->vio.completion, + vio_completion_dump_buffer, + sizeof(vio_completion_dump_buffer)); + if (data_vio->is_duplicate) { + snprintf(vio_block_number_dump_buffer, + sizeof(vio_block_number_dump_buffer), "P%llu L%llu D%llu", + data_vio->allocation.pbn, data_vio->logical.lbn, + data_vio->duplicate.pbn); + } else if (data_vio_has_allocation(data_vio)) { + snprintf(vio_block_number_dump_buffer, + sizeof(vio_block_number_dump_buffer), "P%llu L%llu", + data_vio->allocation.pbn, data_vio->logical.lbn); + } else { + snprintf(vio_block_number_dump_buffer, + sizeof(vio_block_number_dump_buffer), "L%llu", + data_vio->logical.lbn); + } + + if (data_vio->flush_generation != 0) { + snprintf(vio_flush_generation_buffer, + sizeof(vio_flush_generation_buffer), " FG%llu", + data_vio->flush_generation); + } else { + vio_flush_generation_buffer[0] = 0; + } + + encode_vio_dump_flags(data_vio, flags_dump_buffer); + + vdo_log_info(" vio %px %s%s %s %s%s", data_vio, + vio_block_number_dump_buffer, + vio_flush_generation_buffer, + get_data_vio_operation_name(data_vio), + vio_completion_dump_buffer, + flags_dump_buffer); + /* + * might want info on: wantUDSAnswer / operation / status + * might want info on: bio / bios_merged + */ + + dump_vio_waiters(&data_vio->logical.waiters, "lbn"); + + /* might want to dump more info from vio here */ +} diff --git a/drivers/md/dm-vdo/dump.h b/drivers/md/dm-vdo/dump.h new file mode 100644 index 000000000000..ad47c70cca78 --- /dev/null +++ b/drivers/md/dm-vdo/dump.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_DUMP_H +#define VDO_DUMP_H + +#include "types.h" + +int vdo_dump(struct vdo *vdo, unsigned int argc, char *const *argv, const char *why); + +void vdo_dump_all(struct vdo *vdo, const char *why); + +void dump_data_vio(void *data); + +#endif /* VDO_DUMP_H */ diff --git a/drivers/md/dm-vdo/encodings.c b/drivers/md/dm-vdo/encodings.c new file mode 100644 index 000000000000..a34ea0229d53 --- /dev/null +++ b/drivers/md/dm-vdo/encodings.c @@ -0,0 +1,1483 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "encodings.h" + +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "constants.h" +#include "status-codes.h" +#include "types.h" + +/** The maximum logical space is 4 petabytes, which is 1 terablock. */ +static const block_count_t MAXIMUM_VDO_LOGICAL_BLOCKS = 1024ULL * 1024 * 1024 * 1024; + +/** The maximum physical space is 256 terabytes, which is 64 gigablocks. */ +static const block_count_t MAXIMUM_VDO_PHYSICAL_BLOCKS = 1024ULL * 1024 * 1024 * 64; + +struct geometry_block { + char magic_number[VDO_GEOMETRY_MAGIC_NUMBER_SIZE]; + struct packed_header header; + u32 checksum; +} __packed; + +static const struct header GEOMETRY_BLOCK_HEADER_5_0 = { + .id = VDO_GEOMETRY_BLOCK, + .version = { + .major_version = 5, + .minor_version = 0, + }, + /* + * Note: this size isn't just the payload size following the header, like it is everywhere + * else in VDO. + */ + .size = sizeof(struct geometry_block) + sizeof(struct volume_geometry), +}; + +static const struct header GEOMETRY_BLOCK_HEADER_4_0 = { + .id = VDO_GEOMETRY_BLOCK, + .version = { + .major_version = 4, + .minor_version = 0, + }, + /* + * Note: this size isn't just the payload size following the header, like it is everywhere + * else in VDO. + */ + .size = sizeof(struct geometry_block) + sizeof(struct volume_geometry_4_0), +}; + +const u8 VDO_GEOMETRY_MAGIC_NUMBER[VDO_GEOMETRY_MAGIC_NUMBER_SIZE + 1] = "dmvdo001"; + +#define PAGE_HEADER_4_1_SIZE (8 + 8 + 8 + 1 + 1 + 1 + 1) + +static const struct version_number BLOCK_MAP_4_1 = { + .major_version = 4, + .minor_version = 1, +}; + +const struct header VDO_BLOCK_MAP_HEADER_2_0 = { + .id = VDO_BLOCK_MAP, + .version = { + .major_version = 2, + .minor_version = 0, + }, + .size = sizeof(struct block_map_state_2_0), +}; + +const struct header VDO_RECOVERY_JOURNAL_HEADER_7_0 = { + .id = VDO_RECOVERY_JOURNAL, + .version = { + .major_version = 7, + .minor_version = 0, + }, + .size = sizeof(struct recovery_journal_state_7_0), +}; + +const struct header VDO_SLAB_DEPOT_HEADER_2_0 = { + .id = VDO_SLAB_DEPOT, + .version = { + .major_version = 2, + .minor_version = 0, + }, + .size = sizeof(struct slab_depot_state_2_0), +}; + +static const struct header VDO_LAYOUT_HEADER_3_0 = { + .id = VDO_LAYOUT, + .version = { + .major_version = 3, + .minor_version = 0, + }, + .size = sizeof(struct layout_3_0) + (sizeof(struct partition_3_0) * VDO_PARTITION_COUNT), +}; + +static const enum partition_id REQUIRED_PARTITIONS[] = { + VDO_BLOCK_MAP_PARTITION, + VDO_SLAB_DEPOT_PARTITION, + VDO_RECOVERY_JOURNAL_PARTITION, + VDO_SLAB_SUMMARY_PARTITION, +}; + +/* + * The current version for the data encoded in the super block. This must be changed any time there + * is a change to encoding of the component data of any VDO component. + */ +static const struct version_number VDO_COMPONENT_DATA_41_0 = { + .major_version = 41, + .minor_version = 0, +}; + +const struct version_number VDO_VOLUME_VERSION_67_0 = { + .major_version = 67, + .minor_version = 0, +}; + +static const struct header SUPER_BLOCK_HEADER_12_0 = { + .id = VDO_SUPER_BLOCK, + .version = { + .major_version = 12, + .minor_version = 0, + }, + + /* This is the minimum size, if the super block contains no components. */ + .size = VDO_SUPER_BLOCK_FIXED_SIZE - VDO_ENCODED_HEADER_SIZE, +}; + +/** + * validate_version() - Check whether a version matches an expected version. + * @expected_version: The expected version. + * @actual_version: The version being validated. + * @component_name: The name of the component or the calling function (for error logging). + * + * Logs an error describing a mismatch. + * + * Return: VDO_SUCCESS if the versions are the same, + * VDO_UNSUPPORTED_VERSION if the versions don't match. + */ +static int __must_check validate_version(struct version_number expected_version, + struct version_number actual_version, + const char *component_name) +{ + if (!vdo_are_same_version(expected_version, actual_version)) { + return vdo_log_error_strerror(VDO_UNSUPPORTED_VERSION, + "%s version mismatch, expected %d.%d, got %d.%d", + component_name, + expected_version.major_version, + expected_version.minor_version, + actual_version.major_version, + actual_version.minor_version); + } + + return VDO_SUCCESS; +} + +/** + * vdo_validate_header() - Check whether a header matches expectations. + * @expected_header: The expected header. + * @actual_header: The header being validated. + * @exact_size: If true, the size fields of the two headers must be the same, otherwise it is + * required that actual_header.size >= expected_header.size. + * @name: The name of the component or the calling function (for error logging). + * + * Logs an error describing the first mismatch found. + * + * Return: VDO_SUCCESS if the header meets expectations, + * VDO_INCORRECT_COMPONENT if the component ids don't match, + * VDO_UNSUPPORTED_VERSION if the versions or sizes don't match. + */ +int vdo_validate_header(const struct header *expected_header, + const struct header *actual_header, bool exact_size, + const char *name) +{ + int result; + + if (expected_header->id != actual_header->id) { + return vdo_log_error_strerror(VDO_INCORRECT_COMPONENT, + "%s ID mismatch, expected %d, got %d", + name, expected_header->id, + actual_header->id); + } + + result = validate_version(expected_header->version, actual_header->version, + name); + if (result != VDO_SUCCESS) + return result; + + if ((expected_header->size > actual_header->size) || + (exact_size && (expected_header->size < actual_header->size))) { + return vdo_log_error_strerror(VDO_UNSUPPORTED_VERSION, + "%s size mismatch, expected %zu, got %zu", + name, expected_header->size, + actual_header->size); + } + + return VDO_SUCCESS; +} + +static void encode_version_number(u8 *buffer, size_t *offset, + struct version_number version) +{ + struct packed_version_number packed = vdo_pack_version_number(version); + + memcpy(buffer + *offset, &packed, sizeof(packed)); + *offset += sizeof(packed); +} + +void vdo_encode_header(u8 *buffer, size_t *offset, const struct header *header) +{ + struct packed_header packed = vdo_pack_header(header); + + memcpy(buffer + *offset, &packed, sizeof(packed)); + *offset += sizeof(packed); +} + +static void decode_version_number(u8 *buffer, size_t *offset, + struct version_number *version) +{ + struct packed_version_number packed; + + memcpy(&packed, buffer + *offset, sizeof(packed)); + *offset += sizeof(packed); + *version = vdo_unpack_version_number(packed); +} + +void vdo_decode_header(u8 *buffer, size_t *offset, struct header *header) +{ + struct packed_header packed; + + memcpy(&packed, buffer + *offset, sizeof(packed)); + *offset += sizeof(packed); + + *header = vdo_unpack_header(&packed); +} + +/** + * decode_volume_geometry() - Decode the on-disk representation of a volume geometry from a buffer. + * @buffer: A buffer to decode from. + * @offset: The offset in the buffer at which to decode. + * @geometry: The structure to receive the decoded fields. + * @version: The geometry block version to decode. + */ +static void decode_volume_geometry(u8 *buffer, size_t *offset, + struct volume_geometry *geometry, u32 version) +{ + u32 unused, mem; + enum volume_region_id id; + nonce_t nonce; + block_count_t bio_offset = 0; + bool sparse; + + /* This is for backwards compatibility. */ + decode_u32_le(buffer, offset, &unused); + geometry->unused = unused; + + decode_u64_le(buffer, offset, &nonce); + geometry->nonce = nonce; + + memcpy((unsigned char *) &geometry->uuid, buffer + *offset, sizeof(uuid_t)); + *offset += sizeof(uuid_t); + + if (version > 4) + decode_u64_le(buffer, offset, &bio_offset); + geometry->bio_offset = bio_offset; + + for (id = 0; id < VDO_VOLUME_REGION_COUNT; id++) { + physical_block_number_t start_block; + enum volume_region_id saved_id; + + decode_u32_le(buffer, offset, &saved_id); + decode_u64_le(buffer, offset, &start_block); + + geometry->regions[id] = (struct volume_region) { + .id = saved_id, + .start_block = start_block, + }; + } + + decode_u32_le(buffer, offset, &mem); + *offset += sizeof(u32); + sparse = buffer[(*offset)++]; + + geometry->index_config = (struct index_config) { + .mem = mem, + .sparse = sparse, + }; +} + +/** + * vdo_parse_geometry_block() - Decode and validate an encoded geometry block. + * @block: The encoded geometry block. + * @geometry: The structure to receive the decoded fields. + */ +int __must_check vdo_parse_geometry_block(u8 *block, struct volume_geometry *geometry) +{ + u32 checksum, saved_checksum; + struct header header; + size_t offset = 0; + int result; + + if (memcmp(block, VDO_GEOMETRY_MAGIC_NUMBER, VDO_GEOMETRY_MAGIC_NUMBER_SIZE) != 0) + return VDO_BAD_MAGIC; + offset += VDO_GEOMETRY_MAGIC_NUMBER_SIZE; + + vdo_decode_header(block, &offset, &header); + if (header.version.major_version <= 4) { + result = vdo_validate_header(&GEOMETRY_BLOCK_HEADER_4_0, &header, + true, __func__); + } else { + result = vdo_validate_header(&GEOMETRY_BLOCK_HEADER_5_0, &header, + true, __func__); + } + if (result != VDO_SUCCESS) + return result; + + decode_volume_geometry(block, &offset, geometry, header.version.major_version); + + result = VDO_ASSERT(header.size == offset + sizeof(u32), + "should have decoded up to the geometry checksum"); + if (result != VDO_SUCCESS) + return result; + + /* Decode and verify the checksum. */ + checksum = vdo_crc32(block, offset); + decode_u32_le(block, &offset, &saved_checksum); + + return ((checksum == saved_checksum) ? VDO_SUCCESS : VDO_CHECKSUM_MISMATCH); +} + +struct block_map_page *vdo_format_block_map_page(void *buffer, nonce_t nonce, + physical_block_number_t pbn, + bool initialized) +{ + struct block_map_page *page = buffer; + + memset(buffer, 0, VDO_BLOCK_SIZE); + page->version = vdo_pack_version_number(BLOCK_MAP_4_1); + page->header.nonce = __cpu_to_le64(nonce); + page->header.pbn = __cpu_to_le64(pbn); + page->header.initialized = initialized; + return page; +} + +enum block_map_page_validity vdo_validate_block_map_page(struct block_map_page *page, + nonce_t nonce, + physical_block_number_t pbn) +{ + BUILD_BUG_ON(sizeof(struct block_map_page_header) != PAGE_HEADER_4_1_SIZE); + + if (!vdo_are_same_version(BLOCK_MAP_4_1, + vdo_unpack_version_number(page->version)) || + !page->header.initialized || (nonce != __le64_to_cpu(page->header.nonce))) + return VDO_BLOCK_MAP_PAGE_INVALID; + + if (pbn != vdo_get_block_map_page_pbn(page)) + return VDO_BLOCK_MAP_PAGE_BAD; + + return VDO_BLOCK_MAP_PAGE_VALID; +} + +static int decode_block_map_state_2_0(u8 *buffer, size_t *offset, + struct block_map_state_2_0 *state) +{ + size_t initial_offset; + block_count_t flat_page_count, root_count; + physical_block_number_t flat_page_origin, root_origin; + struct header header; + int result; + + vdo_decode_header(buffer, offset, &header); + result = vdo_validate_header(&VDO_BLOCK_MAP_HEADER_2_0, &header, true, __func__); + if (result != VDO_SUCCESS) + return result; + + initial_offset = *offset; + + decode_u64_le(buffer, offset, &flat_page_origin); + result = VDO_ASSERT(flat_page_origin == VDO_BLOCK_MAP_FLAT_PAGE_ORIGIN, + "Flat page origin must be %u (recorded as %llu)", + VDO_BLOCK_MAP_FLAT_PAGE_ORIGIN, + (unsigned long long) state->flat_page_origin); + if (result != VDO_SUCCESS) + return result; + + decode_u64_le(buffer, offset, &flat_page_count); + result = VDO_ASSERT(flat_page_count == 0, + "Flat page count must be 0 (recorded as %llu)", + (unsigned long long) state->flat_page_count); + if (result != VDO_SUCCESS) + return result; + + decode_u64_le(buffer, offset, &root_origin); + decode_u64_le(buffer, offset, &root_count); + + result = VDO_ASSERT(VDO_BLOCK_MAP_HEADER_2_0.size == *offset - initial_offset, + "decoded block map component size must match header size"); + if (result != VDO_SUCCESS) + return result; + + *state = (struct block_map_state_2_0) { + .flat_page_origin = flat_page_origin, + .flat_page_count = flat_page_count, + .root_origin = root_origin, + .root_count = root_count, + }; + + return VDO_SUCCESS; +} + +static void encode_block_map_state_2_0(u8 *buffer, size_t *offset, + struct block_map_state_2_0 state) +{ + size_t initial_offset; + + vdo_encode_header(buffer, offset, &VDO_BLOCK_MAP_HEADER_2_0); + + initial_offset = *offset; + encode_u64_le(buffer, offset, state.flat_page_origin); + encode_u64_le(buffer, offset, state.flat_page_count); + encode_u64_le(buffer, offset, state.root_origin); + encode_u64_le(buffer, offset, state.root_count); + + VDO_ASSERT_LOG_ONLY(VDO_BLOCK_MAP_HEADER_2_0.size == *offset - initial_offset, + "encoded block map component size must match header size"); +} + +/** + * vdo_compute_new_forest_pages() - Compute the number of pages which must be allocated at each + * level in order to grow the forest to a new number of entries. + * @entries: The new number of entries the block map must address. + * + * Return: The total number of non-leaf pages required. + */ +block_count_t vdo_compute_new_forest_pages(root_count_t root_count, + struct boundary *old_sizes, + block_count_t entries, + struct boundary *new_sizes) +{ + page_count_t leaf_pages = max(vdo_compute_block_map_page_count(entries), 1U); + page_count_t level_size = DIV_ROUND_UP(leaf_pages, root_count); + block_count_t total_pages = 0; + height_t height; + + for (height = 0; height < VDO_BLOCK_MAP_TREE_HEIGHT; height++) { + block_count_t new_pages; + + level_size = DIV_ROUND_UP(level_size, VDO_BLOCK_MAP_ENTRIES_PER_PAGE); + new_sizes->levels[height] = level_size; + new_pages = level_size; + if (old_sizes != NULL) + new_pages -= old_sizes->levels[height]; + total_pages += (new_pages * root_count); + } + + return total_pages; +} + +/** + * encode_recovery_journal_state_7_0() - Encode the state of a recovery journal. + * + * Return: VDO_SUCCESS or an error code. + */ +static void encode_recovery_journal_state_7_0(u8 *buffer, size_t *offset, + struct recovery_journal_state_7_0 state) +{ + size_t initial_offset; + + vdo_encode_header(buffer, offset, &VDO_RECOVERY_JOURNAL_HEADER_7_0); + + initial_offset = *offset; + encode_u64_le(buffer, offset, state.journal_start); + encode_u64_le(buffer, offset, state.logical_blocks_used); + encode_u64_le(buffer, offset, state.block_map_data_blocks); + + VDO_ASSERT_LOG_ONLY(VDO_RECOVERY_JOURNAL_HEADER_7_0.size == *offset - initial_offset, + "encoded recovery journal component size must match header size"); +} + +/** + * decode_recovery_journal_state_7_0() - Decode the state of a recovery journal saved in a buffer. + * @buffer: The buffer containing the saved state. + * @state: A pointer to a recovery journal state to hold the result of a successful decode. + * + * Return: VDO_SUCCESS or an error code. + */ +static int __must_check decode_recovery_journal_state_7_0(u8 *buffer, size_t *offset, + struct recovery_journal_state_7_0 *state) +{ + struct header header; + int result; + size_t initial_offset; + sequence_number_t journal_start; + block_count_t logical_blocks_used, block_map_data_blocks; + + vdo_decode_header(buffer, offset, &header); + result = vdo_validate_header(&VDO_RECOVERY_JOURNAL_HEADER_7_0, &header, true, + __func__); + if (result != VDO_SUCCESS) + return result; + + initial_offset = *offset; + decode_u64_le(buffer, offset, &journal_start); + decode_u64_le(buffer, offset, &logical_blocks_used); + decode_u64_le(buffer, offset, &block_map_data_blocks); + + result = VDO_ASSERT(VDO_RECOVERY_JOURNAL_HEADER_7_0.size == *offset - initial_offset, + "decoded recovery journal component size must match header size"); + if (result != VDO_SUCCESS) + return result; + + *state = (struct recovery_journal_state_7_0) { + .journal_start = journal_start, + .logical_blocks_used = logical_blocks_used, + .block_map_data_blocks = block_map_data_blocks, + }; + + return VDO_SUCCESS; +} + +/** + * vdo_get_journal_operation_name() - Get the name of a journal operation. + * @operation: The operation to name. + * + * Return: The name of the operation. + */ +const char *vdo_get_journal_operation_name(enum journal_operation operation) +{ + switch (operation) { + case VDO_JOURNAL_DATA_REMAPPING: + return "data remapping"; + + case VDO_JOURNAL_BLOCK_MAP_REMAPPING: + return "block map remapping"; + + default: + return "unknown journal operation"; + } +} + +/** + * encode_slab_depot_state_2_0() - Encode the state of a slab depot into a buffer. + */ +static void encode_slab_depot_state_2_0(u8 *buffer, size_t *offset, + struct slab_depot_state_2_0 state) +{ + size_t initial_offset; + + vdo_encode_header(buffer, offset, &VDO_SLAB_DEPOT_HEADER_2_0); + + initial_offset = *offset; + encode_u64_le(buffer, offset, state.slab_config.slab_blocks); + encode_u64_le(buffer, offset, state.slab_config.data_blocks); + encode_u64_le(buffer, offset, state.slab_config.reference_count_blocks); + encode_u64_le(buffer, offset, state.slab_config.slab_journal_blocks); + encode_u64_le(buffer, offset, state.slab_config.slab_journal_flushing_threshold); + encode_u64_le(buffer, offset, state.slab_config.slab_journal_blocking_threshold); + encode_u64_le(buffer, offset, state.slab_config.slab_journal_scrubbing_threshold); + encode_u64_le(buffer, offset, state.first_block); + encode_u64_le(buffer, offset, state.last_block); + buffer[(*offset)++] = state.zone_count; + + VDO_ASSERT_LOG_ONLY(VDO_SLAB_DEPOT_HEADER_2_0.size == *offset - initial_offset, + "encoded block map component size must match header size"); +} + +/** + * decode_slab_depot_state_2_0() - Decode slab depot component state version 2.0 from a buffer. + * + * Return: VDO_SUCCESS or an error code. + */ +static int decode_slab_depot_state_2_0(u8 *buffer, size_t *offset, + struct slab_depot_state_2_0 *state) +{ + struct header header; + int result; + size_t initial_offset; + struct slab_config slab_config; + block_count_t count; + physical_block_number_t first_block, last_block; + zone_count_t zone_count; + + vdo_decode_header(buffer, offset, &header); + result = vdo_validate_header(&VDO_SLAB_DEPOT_HEADER_2_0, &header, true, + __func__); + if (result != VDO_SUCCESS) + return result; + + initial_offset = *offset; + decode_u64_le(buffer, offset, &count); + slab_config.slab_blocks = count; + + decode_u64_le(buffer, offset, &count); + slab_config.data_blocks = count; + + decode_u64_le(buffer, offset, &count); + slab_config.reference_count_blocks = count; + + decode_u64_le(buffer, offset, &count); + slab_config.slab_journal_blocks = count; + + decode_u64_le(buffer, offset, &count); + slab_config.slab_journal_flushing_threshold = count; + + decode_u64_le(buffer, offset, &count); + slab_config.slab_journal_blocking_threshold = count; + + decode_u64_le(buffer, offset, &count); + slab_config.slab_journal_scrubbing_threshold = count; + + decode_u64_le(buffer, offset, &first_block); + decode_u64_le(buffer, offset, &last_block); + zone_count = buffer[(*offset)++]; + + result = VDO_ASSERT(VDO_SLAB_DEPOT_HEADER_2_0.size == *offset - initial_offset, + "decoded slab depot component size must match header size"); + if (result != VDO_SUCCESS) + return result; + + *state = (struct slab_depot_state_2_0) { + .slab_config = slab_config, + .first_block = first_block, + .last_block = last_block, + .zone_count = zone_count, + }; + + return VDO_SUCCESS; +} + +/** + * vdo_configure_slab_depot() - Configure the slab depot. + * @partition: The slab depot partition + * @slab_config: The configuration of a single slab. + * @zone_count: The number of zones the depot will use. + * @state: The state structure to be configured. + * + * Configures the slab_depot for the specified storage capacity, finding the number of data blocks + * that will fit and still leave room for the depot metadata, then return the saved state for that + * configuration. + * + * Return: VDO_SUCCESS or an error code. + */ +int vdo_configure_slab_depot(const struct partition *partition, + struct slab_config slab_config, zone_count_t zone_count, + struct slab_depot_state_2_0 *state) +{ + block_count_t total_slab_blocks, total_data_blocks; + size_t slab_count; + physical_block_number_t last_block; + block_count_t slab_size = slab_config.slab_blocks; + + vdo_log_debug("slabDepot %s(block_count=%llu, first_block=%llu, slab_size=%llu, zone_count=%u)", + __func__, (unsigned long long) partition->count, + (unsigned long long) partition->offset, + (unsigned long long) slab_size, zone_count); + + /* We do not allow runt slabs, so we waste up to a slab's worth. */ + slab_count = (partition->count / slab_size); + if (slab_count == 0) + return VDO_NO_SPACE; + + if (slab_count > MAX_VDO_SLABS) + return VDO_TOO_MANY_SLABS; + + total_slab_blocks = slab_count * slab_config.slab_blocks; + total_data_blocks = slab_count * slab_config.data_blocks; + last_block = partition->offset + total_slab_blocks; + + *state = (struct slab_depot_state_2_0) { + .slab_config = slab_config, + .first_block = partition->offset, + .last_block = last_block, + .zone_count = zone_count, + }; + + vdo_log_debug("slab_depot last_block=%llu, total_data_blocks=%llu, slab_count=%zu, left_over=%llu", + (unsigned long long) last_block, + (unsigned long long) total_data_blocks, slab_count, + (unsigned long long) (partition->count - (last_block - partition->offset))); + + return VDO_SUCCESS; +} + +/** + * vdo_configure_slab() - Measure and initialize the configuration to use for each slab. + * @slab_size: The number of blocks per slab. + * @slab_journal_blocks: The number of blocks for the slab journal. + * @slab_config: The slab configuration to initialize. + * + * Return: VDO_SUCCESS or an error code. + */ +int vdo_configure_slab(block_count_t slab_size, block_count_t slab_journal_blocks, + struct slab_config *slab_config) +{ + block_count_t ref_blocks, meta_blocks, data_blocks; + block_count_t flushing_threshold, remaining, blocking_threshold; + block_count_t minimal_extra_space, scrubbing_threshold; + + if (slab_journal_blocks >= slab_size) + return VDO_BAD_CONFIGURATION; + + /* + * This calculation should technically be a recurrence, but the total number of metadata + * blocks is currently less than a single block of ref_counts, so we'd gain at most one + * data block in each slab with more iteration. + */ + ref_blocks = vdo_get_saved_reference_count_size(slab_size - slab_journal_blocks); + meta_blocks = (ref_blocks + slab_journal_blocks); + + /* Make sure test code hasn't configured slabs to be too small. */ + if (meta_blocks >= slab_size) + return VDO_BAD_CONFIGURATION; + + /* + * If the slab size is very small, assume this must be a unit test and override the number + * of data blocks to be a power of two (wasting blocks in the slab). Many tests need their + * data_blocks fields to be the exact capacity of the configured volume, and that used to + * fall out since they use a power of two for the number of data blocks, the slab size was + * a power of two, and every block in a slab was a data block. + * + * TODO: Try to figure out some way of structuring testParameters and unit tests so this + * hack isn't needed without having to edit several unit tests every time the metadata size + * changes by one block. + */ + data_blocks = slab_size - meta_blocks; + if ((slab_size < 1024) && !is_power_of_2(data_blocks)) + data_blocks = ((block_count_t) 1 << ilog2(data_blocks)); + + /* + * Configure the slab journal thresholds. The flush threshold is 168 of 224 blocks in + * production, or 3/4ths, so we use this ratio for all sizes. + */ + flushing_threshold = ((slab_journal_blocks * 3) + 3) / 4; + /* + * The blocking threshold should be far enough from the flushing threshold to not produce + * delays, but far enough from the end of the journal to allow multiple successive recovery + * failures. + */ + remaining = slab_journal_blocks - flushing_threshold; + blocking_threshold = flushing_threshold + ((remaining * 5) / 7); + /* The scrubbing threshold should be at least 2048 entries before the end of the journal. */ + minimal_extra_space = 1 + (MAXIMUM_VDO_USER_VIOS / VDO_SLAB_JOURNAL_FULL_ENTRIES_PER_BLOCK); + scrubbing_threshold = blocking_threshold; + if (slab_journal_blocks > minimal_extra_space) + scrubbing_threshold = slab_journal_blocks - minimal_extra_space; + if (blocking_threshold > scrubbing_threshold) + blocking_threshold = scrubbing_threshold; + + *slab_config = (struct slab_config) { + .slab_blocks = slab_size, + .data_blocks = data_blocks, + .reference_count_blocks = ref_blocks, + .slab_journal_blocks = slab_journal_blocks, + .slab_journal_flushing_threshold = flushing_threshold, + .slab_journal_blocking_threshold = blocking_threshold, + .slab_journal_scrubbing_threshold = scrubbing_threshold}; + return VDO_SUCCESS; +} + +/** + * vdo_decode_slab_journal_entry() - Decode a slab journal entry. + * @block: The journal block holding the entry. + * @entry_count: The number of the entry. + * + * Return: The decoded entry. + */ +struct slab_journal_entry vdo_decode_slab_journal_entry(struct packed_slab_journal_block *block, + journal_entry_count_t entry_count) +{ + struct slab_journal_entry entry = + vdo_unpack_slab_journal_entry(&block->payload.entries[entry_count]); + + if (block->header.has_block_map_increments && + ((block->payload.full_entries.entry_types[entry_count / 8] & + ((u8) 1 << (entry_count % 8))) != 0)) + entry.operation = VDO_JOURNAL_BLOCK_MAP_REMAPPING; + + return entry; +} + +/** + * allocate_partition() - Allocate a partition and add it to a layout. + * @layout: The layout containing the partition. + * @id: The id of the partition. + * @offset: The offset into the layout at which the partition begins. + * @size: The size of the partition in blocks. + * + * Return: VDO_SUCCESS or an error. + */ +static int allocate_partition(struct layout *layout, u8 id, + physical_block_number_t offset, block_count_t size) +{ + struct partition *partition; + int result; + + result = vdo_allocate(1, struct partition, __func__, &partition); + if (result != VDO_SUCCESS) + return result; + + partition->id = id; + partition->offset = offset; + partition->count = size; + partition->next = layout->head; + layout->head = partition; + + return VDO_SUCCESS; +} + +/** + * make_partition() - Create a new partition from the beginning or end of the unused space in a + * layout. + * @layout: The layout. + * @id: The id of the partition to make. + * @size: The number of blocks to carve out; if 0, all remaining space will be used. + * @beginning: True if the partition should start at the beginning of the unused space. + * + * Return: A success or error code, particularly VDO_NO_SPACE if there are fewer than size blocks + * remaining. + */ +static int __must_check make_partition(struct layout *layout, enum partition_id id, + block_count_t size, bool beginning) +{ + int result; + physical_block_number_t offset; + block_count_t free_blocks = layout->last_free - layout->first_free; + + if (size == 0) { + if (free_blocks == 0) + return VDO_NO_SPACE; + size = free_blocks; + } else if (size > free_blocks) { + return VDO_NO_SPACE; + } + + result = vdo_get_partition(layout, id, NULL); + if (result != VDO_UNKNOWN_PARTITION) + return VDO_PARTITION_EXISTS; + + offset = beginning ? layout->first_free : (layout->last_free - size); + + result = allocate_partition(layout, id, offset, size); + if (result != VDO_SUCCESS) + return result; + + layout->num_partitions++; + if (beginning) + layout->first_free += size; + else + layout->last_free = layout->last_free - size; + + return VDO_SUCCESS; +} + +/** + * vdo_initialize_layout() - Lay out the partitions of a vdo. + * @size: The entire size of the vdo. + * @origin: The start of the layout on the underlying storage in blocks. + * @block_map_blocks: The size of the block map partition. + * @journal_blocks: The size of the journal partition. + * @summary_blocks: The size of the slab summary partition. + * @layout: The layout to initialize. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_initialize_layout(block_count_t size, physical_block_number_t offset, + block_count_t block_map_blocks, block_count_t journal_blocks, + block_count_t summary_blocks, struct layout *layout) +{ + int result; + block_count_t necessary_size = + (offset + block_map_blocks + journal_blocks + summary_blocks); + + if (necessary_size > size) + return vdo_log_error_strerror(VDO_NO_SPACE, + "Not enough space to make a VDO"); + + *layout = (struct layout) { + .start = offset, + .size = size, + .first_free = offset, + .last_free = size, + .num_partitions = 0, + .head = NULL, + }; + + result = make_partition(layout, VDO_BLOCK_MAP_PARTITION, block_map_blocks, true); + if (result != VDO_SUCCESS) { + vdo_uninitialize_layout(layout); + return result; + } + + result = make_partition(layout, VDO_SLAB_SUMMARY_PARTITION, summary_blocks, + false); + if (result != VDO_SUCCESS) { + vdo_uninitialize_layout(layout); + return result; + } + + result = make_partition(layout, VDO_RECOVERY_JOURNAL_PARTITION, journal_blocks, + false); + if (result != VDO_SUCCESS) { + vdo_uninitialize_layout(layout); + return result; + } + + result = make_partition(layout, VDO_SLAB_DEPOT_PARTITION, 0, true); + if (result != VDO_SUCCESS) + vdo_uninitialize_layout(layout); + + return result; +} + +/** + * vdo_uninitialize_layout() - Clean up a layout. + * @layout: The layout to clean up. + * + * All partitions created by this layout become invalid pointers. + */ +void vdo_uninitialize_layout(struct layout *layout) +{ + while (layout->head != NULL) { + struct partition *part = layout->head; + + layout->head = part->next; + vdo_free(part); + } + + memset(layout, 0, sizeof(struct layout)); +} + +/** + * vdo_get_partition() - Get a partition by id. + * @layout: The layout from which to get a partition. + * @id: The id of the partition. + * @partition_ptr: A pointer to hold the partition. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_get_partition(struct layout *layout, enum partition_id id, + struct partition **partition_ptr) +{ + struct partition *partition; + + for (partition = layout->head; partition != NULL; partition = partition->next) { + if (partition->id == id) { + if (partition_ptr != NULL) + *partition_ptr = partition; + return VDO_SUCCESS; + } + } + + return VDO_UNKNOWN_PARTITION; +} + +/** + * vdo_get_known_partition() - Get a partition by id from a validated layout. + * @layout: The layout from which to get a partition. + * @id: The id of the partition. + * + * Return: the partition + */ +struct partition *vdo_get_known_partition(struct layout *layout, enum partition_id id) +{ + struct partition *partition; + int result = vdo_get_partition(layout, id, &partition); + + VDO_ASSERT_LOG_ONLY(result == VDO_SUCCESS, "layout has expected partition: %u", id); + + return partition; +} + +static void encode_layout(u8 *buffer, size_t *offset, const struct layout *layout) +{ + const struct partition *partition; + size_t initial_offset; + struct header header = VDO_LAYOUT_HEADER_3_0; + + BUILD_BUG_ON(sizeof(enum partition_id) != sizeof(u8)); + VDO_ASSERT_LOG_ONLY(layout->num_partitions <= U8_MAX, + "layout partition count must fit in a byte"); + + vdo_encode_header(buffer, offset, &header); + + initial_offset = *offset; + encode_u64_le(buffer, offset, layout->first_free); + encode_u64_le(buffer, offset, layout->last_free); + buffer[(*offset)++] = layout->num_partitions; + + VDO_ASSERT_LOG_ONLY(sizeof(struct layout_3_0) == *offset - initial_offset, + "encoded size of a layout header must match structure"); + + for (partition = layout->head; partition != NULL; partition = partition->next) { + buffer[(*offset)++] = partition->id; + encode_u64_le(buffer, offset, partition->offset); + /* This field only exists for backwards compatibility */ + encode_u64_le(buffer, offset, 0); + encode_u64_le(buffer, offset, partition->count); + } + + VDO_ASSERT_LOG_ONLY(header.size == *offset - initial_offset, + "encoded size of a layout must match header size"); +} + +static int decode_layout(u8 *buffer, size_t *offset, physical_block_number_t start, + block_count_t size, struct layout *layout) +{ + struct header header; + struct layout_3_0 layout_header; + struct partition *partition; + size_t initial_offset; + physical_block_number_t first_free, last_free; + u8 partition_count; + u8 i; + int result; + + vdo_decode_header(buffer, offset, &header); + /* Layout is variable size, so only do a minimum size check here. */ + result = vdo_validate_header(&VDO_LAYOUT_HEADER_3_0, &header, false, __func__); + if (result != VDO_SUCCESS) + return result; + + initial_offset = *offset; + decode_u64_le(buffer, offset, &first_free); + decode_u64_le(buffer, offset, &last_free); + partition_count = buffer[(*offset)++]; + layout_header = (struct layout_3_0) { + .first_free = first_free, + .last_free = last_free, + .partition_count = partition_count, + }; + + result = VDO_ASSERT(sizeof(struct layout_3_0) == *offset - initial_offset, + "decoded size of a layout header must match structure"); + if (result != VDO_SUCCESS) + return result; + + layout->start = start; + layout->size = size; + layout->first_free = layout_header.first_free; + layout->last_free = layout_header.last_free; + layout->num_partitions = layout_header.partition_count; + + if (layout->num_partitions > VDO_PARTITION_COUNT) { + return vdo_log_error_strerror(VDO_UNKNOWN_PARTITION, + "layout has extra partitions"); + } + + for (i = 0; i < layout->num_partitions; i++) { + u8 id; + u64 partition_offset, count; + + id = buffer[(*offset)++]; + decode_u64_le(buffer, offset, &partition_offset); + *offset += sizeof(u64); + decode_u64_le(buffer, offset, &count); + + result = allocate_partition(layout, id, partition_offset, count); + if (result != VDO_SUCCESS) { + vdo_uninitialize_layout(layout); + return result; + } + } + + /* Validate that the layout has all (and only) the required partitions */ + for (i = 0; i < VDO_PARTITION_COUNT; i++) { + result = vdo_get_partition(layout, REQUIRED_PARTITIONS[i], &partition); + if (result != VDO_SUCCESS) { + vdo_uninitialize_layout(layout); + return vdo_log_error_strerror(result, + "layout is missing required partition %u", + REQUIRED_PARTITIONS[i]); + } + + start += partition->count; + } + + if (start != size) { + vdo_uninitialize_layout(layout); + return vdo_log_error_strerror(UDS_BAD_STATE, + "partitions do not cover the layout"); + } + + return VDO_SUCCESS; +} + +/** + * pack_vdo_config() - Convert a vdo_config to its packed on-disk representation. + * @config: The vdo config to convert. + * + * Return: The platform-independent representation of the config. + */ +static struct packed_vdo_config pack_vdo_config(struct vdo_config config) +{ + return (struct packed_vdo_config) { + .logical_blocks = __cpu_to_le64(config.logical_blocks), + .physical_blocks = __cpu_to_le64(config.physical_blocks), + .slab_size = __cpu_to_le64(config.slab_size), + .recovery_journal_size = __cpu_to_le64(config.recovery_journal_size), + .slab_journal_blocks = __cpu_to_le64(config.slab_journal_blocks), + }; +} + +/** + * pack_vdo_component() - Convert a vdo_component to its packed on-disk representation. + * @component: The VDO component data to convert. + * + * Return: The platform-independent representation of the component. + */ +static struct packed_vdo_component_41_0 pack_vdo_component(const struct vdo_component component) +{ + return (struct packed_vdo_component_41_0) { + .state = __cpu_to_le32(component.state), + .complete_recoveries = __cpu_to_le64(component.complete_recoveries), + .read_only_recoveries = __cpu_to_le64(component.read_only_recoveries), + .config = pack_vdo_config(component.config), + .nonce = __cpu_to_le64(component.nonce), + }; +} + +static void encode_vdo_component(u8 *buffer, size_t *offset, + struct vdo_component component) +{ + struct packed_vdo_component_41_0 packed; + + encode_version_number(buffer, offset, VDO_COMPONENT_DATA_41_0); + packed = pack_vdo_component(component); + memcpy(buffer + *offset, &packed, sizeof(packed)); + *offset += sizeof(packed); +} + +/** + * unpack_vdo_config() - Convert a packed_vdo_config to its native in-memory representation. + * @config: The packed vdo config to convert. + * + * Return: The native in-memory representation of the vdo config. + */ +static struct vdo_config unpack_vdo_config(struct packed_vdo_config config) +{ + return (struct vdo_config) { + .logical_blocks = __le64_to_cpu(config.logical_blocks), + .physical_blocks = __le64_to_cpu(config.physical_blocks), + .slab_size = __le64_to_cpu(config.slab_size), + .recovery_journal_size = __le64_to_cpu(config.recovery_journal_size), + .slab_journal_blocks = __le64_to_cpu(config.slab_journal_blocks), + }; +} + +/** + * unpack_vdo_component_41_0() - Convert a packed_vdo_component_41_0 to its native in-memory + * representation. + * @component: The packed vdo component data to convert. + * + * Return: The native in-memory representation of the component. + */ +static struct vdo_component unpack_vdo_component_41_0(struct packed_vdo_component_41_0 component) +{ + return (struct vdo_component) { + .state = __le32_to_cpu(component.state), + .complete_recoveries = __le64_to_cpu(component.complete_recoveries), + .read_only_recoveries = __le64_to_cpu(component.read_only_recoveries), + .config = unpack_vdo_config(component.config), + .nonce = __le64_to_cpu(component.nonce), + }; +} + +/** + * decode_vdo_component() - Decode the component data for the vdo itself out of the super block. + * + * Return: VDO_SUCCESS or an error. + */ +static int decode_vdo_component(u8 *buffer, size_t *offset, struct vdo_component *component) +{ + struct version_number version; + struct packed_vdo_component_41_0 packed; + int result; + + decode_version_number(buffer, offset, &version); + result = validate_version(version, VDO_COMPONENT_DATA_41_0, + "VDO component data"); + if (result != VDO_SUCCESS) + return result; + + memcpy(&packed, buffer + *offset, sizeof(packed)); + *offset += sizeof(packed); + *component = unpack_vdo_component_41_0(packed); + return VDO_SUCCESS; +} + +/** + * vdo_validate_config() - Validate constraints on a VDO config. + * @config: The VDO config. + * @physical_block_count: The minimum block count of the underlying storage. + * @logical_block_count: The expected logical size of the VDO, or 0 if the logical size may be + * unspecified. + * + * Return: A success or error code. + */ +int vdo_validate_config(const struct vdo_config *config, + block_count_t physical_block_count, + block_count_t logical_block_count) +{ + struct slab_config slab_config; + int result; + + result = VDO_ASSERT(config->slab_size > 0, "slab size unspecified"); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT(is_power_of_2(config->slab_size), + "slab size must be a power of two"); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT(config->slab_size <= (1 << MAX_VDO_SLAB_BITS), + "slab size must be less than or equal to 2^%d", + MAX_VDO_SLAB_BITS); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT(config->slab_journal_blocks >= MINIMUM_VDO_SLAB_JOURNAL_BLOCKS, + "slab journal size meets minimum size"); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT(config->slab_journal_blocks <= config->slab_size, + "slab journal size is within expected bound"); + if (result != VDO_SUCCESS) + return result; + + result = vdo_configure_slab(config->slab_size, config->slab_journal_blocks, + &slab_config); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT((slab_config.data_blocks >= 1), + "slab must be able to hold at least one block"); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT(config->physical_blocks > 0, "physical blocks unspecified"); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT(config->physical_blocks <= MAXIMUM_VDO_PHYSICAL_BLOCKS, + "physical block count %llu exceeds maximum %llu", + (unsigned long long) config->physical_blocks, + (unsigned long long) MAXIMUM_VDO_PHYSICAL_BLOCKS); + if (result != VDO_SUCCESS) + return VDO_OUT_OF_RANGE; + + if (physical_block_count != config->physical_blocks) { + vdo_log_error("A physical size of %llu blocks was specified, not the %llu blocks configured in the vdo super block", + (unsigned long long) physical_block_count, + (unsigned long long) config->physical_blocks); + return VDO_PARAMETER_MISMATCH; + } + + if (logical_block_count > 0) { + result = VDO_ASSERT((config->logical_blocks > 0), + "logical blocks unspecified"); + if (result != VDO_SUCCESS) + return result; + + if (logical_block_count != config->logical_blocks) { + vdo_log_error("A logical size of %llu blocks was specified, but that differs from the %llu blocks configured in the vdo super block", + (unsigned long long) logical_block_count, + (unsigned long long) config->logical_blocks); + return VDO_PARAMETER_MISMATCH; + } + } + + result = VDO_ASSERT(config->logical_blocks <= MAXIMUM_VDO_LOGICAL_BLOCKS, + "logical blocks too large"); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT(config->recovery_journal_size > 0, + "recovery journal size unspecified"); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT(is_power_of_2(config->recovery_journal_size), + "recovery journal size must be a power of two"); + if (result != VDO_SUCCESS) + return result; + + return result; +} + +/** + * vdo_destroy_component_states() - Clean up any allocations in a vdo_component_states. + * @states: The component states to destroy. + */ +void vdo_destroy_component_states(struct vdo_component_states *states) +{ + if (states == NULL) + return; + + vdo_uninitialize_layout(&states->layout); +} + +/** + * decode_components() - Decode the components now that we know the component data is a version we + * understand. + * @buffer: The buffer being decoded. + * @offset: The offset to start decoding from. + * @geometry: The vdo geometry + * @states: An object to hold the successfully decoded state. + * + * Return: VDO_SUCCESS or an error. + */ +static int __must_check decode_components(u8 *buffer, size_t *offset, + struct volume_geometry *geometry, + struct vdo_component_states *states) +{ + int result; + + decode_vdo_component(buffer, offset, &states->vdo); + + result = decode_layout(buffer, offset, vdo_get_data_region_start(*geometry) + 1, + states->vdo.config.physical_blocks, &states->layout); + if (result != VDO_SUCCESS) + return result; + + result = decode_recovery_journal_state_7_0(buffer, offset, + &states->recovery_journal); + if (result != VDO_SUCCESS) + return result; + + result = decode_slab_depot_state_2_0(buffer, offset, &states->slab_depot); + if (result != VDO_SUCCESS) + return result; + + result = decode_block_map_state_2_0(buffer, offset, &states->block_map); + if (result != VDO_SUCCESS) + return result; + + VDO_ASSERT_LOG_ONLY(*offset == VDO_COMPONENT_DATA_OFFSET + VDO_COMPONENT_DATA_SIZE, + "All decoded component data was used"); + return VDO_SUCCESS; +} + +/** + * vdo_decode_component_states() - Decode the payload of a super block. + * @buffer: The buffer containing the encoded super block contents. + * @geometry: The vdo geometry + * @states: A pointer to hold the decoded states. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_decode_component_states(u8 *buffer, struct volume_geometry *geometry, + struct vdo_component_states *states) +{ + int result; + size_t offset = VDO_COMPONENT_DATA_OFFSET; + + /* This is for backwards compatibility. */ + decode_u32_le(buffer, &offset, &states->unused); + + /* Check the VDO volume version */ + decode_version_number(buffer, &offset, &states->volume_version); + result = validate_version(VDO_VOLUME_VERSION_67_0, states->volume_version, + "volume"); + if (result != VDO_SUCCESS) + return result; + + result = decode_components(buffer, &offset, geometry, states); + if (result != VDO_SUCCESS) + vdo_uninitialize_layout(&states->layout); + + return result; +} + +/** + * vdo_validate_component_states() - Validate the decoded super block configuration. + * @states: The state decoded from the super block. + * @geometry_nonce: The nonce from the geometry block. + * @physical_size: The minimum block count of the underlying storage. + * @logical_size: The expected logical size of the VDO, or 0 if the logical size may be + * unspecified. + * + * Return: VDO_SUCCESS or an error if the configuration is invalid. + */ +int vdo_validate_component_states(struct vdo_component_states *states, + nonce_t geometry_nonce, block_count_t physical_size, + block_count_t logical_size) +{ + if (geometry_nonce != states->vdo.nonce) { + return vdo_log_error_strerror(VDO_BAD_NONCE, + "Geometry nonce %llu does not match superblock nonce %llu", + (unsigned long long) geometry_nonce, + (unsigned long long) states->vdo.nonce); + } + + return vdo_validate_config(&states->vdo.config, physical_size, logical_size); +} + +/** + * vdo_encode_component_states() - Encode the state of all vdo components in the super block. + */ +static void vdo_encode_component_states(u8 *buffer, size_t *offset, + const struct vdo_component_states *states) +{ + /* This is for backwards compatibility. */ + encode_u32_le(buffer, offset, states->unused); + encode_version_number(buffer, offset, states->volume_version); + encode_vdo_component(buffer, offset, states->vdo); + encode_layout(buffer, offset, &states->layout); + encode_recovery_journal_state_7_0(buffer, offset, states->recovery_journal); + encode_slab_depot_state_2_0(buffer, offset, states->slab_depot); + encode_block_map_state_2_0(buffer, offset, states->block_map); + + VDO_ASSERT_LOG_ONLY(*offset == VDO_COMPONENT_DATA_OFFSET + VDO_COMPONENT_DATA_SIZE, + "All super block component data was encoded"); +} + +/** + * vdo_encode_super_block() - Encode a super block into its on-disk representation. + */ +void vdo_encode_super_block(u8 *buffer, struct vdo_component_states *states) +{ + u32 checksum; + struct header header = SUPER_BLOCK_HEADER_12_0; + size_t offset = 0; + + header.size += VDO_COMPONENT_DATA_SIZE; + vdo_encode_header(buffer, &offset, &header); + vdo_encode_component_states(buffer, &offset, states); + + checksum = vdo_crc32(buffer, offset); + encode_u32_le(buffer, &offset, checksum); + + /* + * Even though the buffer is a full block, to avoid the potential corruption from a torn + * write, the entire encoding must fit in the first sector. + */ + VDO_ASSERT_LOG_ONLY(offset <= VDO_SECTOR_SIZE, + "entire superblock must fit in one sector"); +} + +/** + * vdo_decode_super_block() - Decode a super block from its on-disk representation. + */ +int vdo_decode_super_block(u8 *buffer) +{ + struct header header; + int result; + u32 checksum, saved_checksum; + size_t offset = 0; + + /* Decode and validate the header. */ + vdo_decode_header(buffer, &offset, &header); + result = vdo_validate_header(&SUPER_BLOCK_HEADER_12_0, &header, false, __func__); + if (result != VDO_SUCCESS) + return result; + + if (header.size > VDO_COMPONENT_DATA_SIZE + sizeof(u32)) { + /* + * We can't check release version or checksum until we know the content size, so we + * have to assume a version mismatch on unexpected values. + */ + return vdo_log_error_strerror(VDO_UNSUPPORTED_VERSION, + "super block contents too large: %zu", + header.size); + } + + /* Skip past the component data for now, to verify the checksum. */ + offset += VDO_COMPONENT_DATA_SIZE; + + checksum = vdo_crc32(buffer, offset); + decode_u32_le(buffer, &offset, &saved_checksum); + + result = VDO_ASSERT(offset == VDO_SUPER_BLOCK_FIXED_SIZE + VDO_COMPONENT_DATA_SIZE, + "must have decoded entire superblock payload"); + if (result != VDO_SUCCESS) + return result; + + return ((checksum != saved_checksum) ? VDO_CHECKSUM_MISMATCH : VDO_SUCCESS); +} diff --git a/drivers/md/dm-vdo/encodings.h b/drivers/md/dm-vdo/encodings.h new file mode 100644 index 000000000000..e5ff2b0aaa79 --- /dev/null +++ b/drivers/md/dm-vdo/encodings.h @@ -0,0 +1,1298 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_ENCODINGS_H +#define VDO_ENCODINGS_H + +#include +#include +#include +#include + +#include "numeric.h" + +#include "constants.h" +#include "types.h" + +/* + * An in-memory representation of a version number for versioned structures on disk. + * + * A version number consists of two portions, a major version and a minor version. Any format + * change which does not require an explicit upgrade step from the previous version should + * increment the minor version. Any format change which either requires an explicit upgrade step, + * or is wholly incompatible (i.e. can not be upgraded to), should increment the major version, and + * set the minor version to 0. + */ +struct version_number { + u32 major_version; + u32 minor_version; +}; + +/* + * A packed, machine-independent, on-disk representation of a version_number. Both fields are + * stored in little-endian byte order. + */ +struct packed_version_number { + __le32 major_version; + __le32 minor_version; +} __packed; + +/* The registry of component ids for use in headers */ +#define VDO_SUPER_BLOCK 0 +#define VDO_LAYOUT 1 +#define VDO_RECOVERY_JOURNAL 2 +#define VDO_SLAB_DEPOT 3 +#define VDO_BLOCK_MAP 4 +#define VDO_GEOMETRY_BLOCK 5 + +/* The header for versioned data stored on disk. */ +struct header { + u32 id; /* The component this is a header for */ + struct version_number version; /* The version of the data format */ + size_t size; /* The size of the data following this header */ +}; + +/* A packed, machine-independent, on-disk representation of a component header. */ +struct packed_header { + __le32 id; + struct packed_version_number version; + __le64 size; +} __packed; + +enum { + VDO_GEOMETRY_BLOCK_LOCATION = 0, + VDO_GEOMETRY_MAGIC_NUMBER_SIZE = 8, + VDO_DEFAULT_GEOMETRY_BLOCK_VERSION = 5, +}; + +struct index_config { + u32 mem; + u32 unused; + bool sparse; +} __packed; + +enum volume_region_id { + VDO_INDEX_REGION = 0, + VDO_DATA_REGION = 1, + VDO_VOLUME_REGION_COUNT, +}; + +struct volume_region { + /* The ID of the region */ + enum volume_region_id id; + /* + * The absolute starting offset on the device. The region continues until the next region + * begins. + */ + physical_block_number_t start_block; +} __packed; + +struct volume_geometry { + /* For backwards compatibility */ + u32 unused; + /* The nonce of this volume */ + nonce_t nonce; + /* The uuid of this volume */ + uuid_t uuid; + /* The block offset to be applied to bios */ + block_count_t bio_offset; + /* The regions in ID order */ + struct volume_region regions[VDO_VOLUME_REGION_COUNT]; + /* The index config */ + struct index_config index_config; +} __packed; + +/* This volume geometry struct is used for sizing only */ +struct volume_geometry_4_0 { + /* For backwards compatibility */ + u32 unused; + /* The nonce of this volume */ + nonce_t nonce; + /* The uuid of this volume */ + uuid_t uuid; + /* The regions in ID order */ + struct volume_region regions[VDO_VOLUME_REGION_COUNT]; + /* The index config */ + struct index_config index_config; +} __packed; + +extern const u8 VDO_GEOMETRY_MAGIC_NUMBER[VDO_GEOMETRY_MAGIC_NUMBER_SIZE + 1]; + +/** + * DOC: Block map entries + * + * The entry for each logical block in the block map is encoded into five bytes, which saves space + * in both the on-disk and in-memory layouts. It consists of the 36 low-order bits of a + * physical_block_number_t (addressing 256 terabytes with a 4KB block size) and a 4-bit encoding of + * a block_mapping_state. + * + * Of the 8 high bits of the 5-byte structure: + * + * Bits 7..4: The four highest bits of the 36-bit physical block number + * Bits 3..0: The 4-bit block_mapping_state + * + * The following 4 bytes are the low order bytes of the physical block number, in little-endian + * order. + * + * Conversion functions to and from a data location are provided. + */ +struct block_map_entry { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + unsigned mapping_state : 4; + unsigned pbn_high_nibble : 4; +#else + unsigned pbn_high_nibble : 4; + unsigned mapping_state : 4; +#endif + + __le32 pbn_low_word; +} __packed; + +struct block_map_page_header { + __le64 nonce; + __le64 pbn; + + /* May be non-zero on disk */ + u8 unused_long_word[8]; + + /* Whether this page has been written twice to disk */ + bool initialized; + + /* Always zero on disk */ + u8 unused_byte1; + + /* May be non-zero on disk */ + u8 unused_byte2; + u8 unused_byte3; +} __packed; + +struct block_map_page { + struct packed_version_number version; + struct block_map_page_header header; + struct block_map_entry entries[]; +} __packed; + +enum block_map_page_validity { + VDO_BLOCK_MAP_PAGE_VALID, + VDO_BLOCK_MAP_PAGE_INVALID, + /* Valid page found in the wrong location on disk */ + VDO_BLOCK_MAP_PAGE_BAD, +}; + +struct block_map_state_2_0 { + physical_block_number_t flat_page_origin; + block_count_t flat_page_count; + physical_block_number_t root_origin; + block_count_t root_count; +} __packed; + +struct boundary { + page_number_t levels[VDO_BLOCK_MAP_TREE_HEIGHT]; +}; + +extern const struct header VDO_BLOCK_MAP_HEADER_2_0; + +/* The state of the recovery journal as encoded in the VDO super block. */ +struct recovery_journal_state_7_0 { + /* Sequence number to start the journal */ + sequence_number_t journal_start; + /* Number of logical blocks used by VDO */ + block_count_t logical_blocks_used; + /* Number of block map pages allocated */ + block_count_t block_map_data_blocks; +} __packed; + +extern const struct header VDO_RECOVERY_JOURNAL_HEADER_7_0; + +typedef u16 journal_entry_count_t; + +/* + * A recovery journal entry stores three physical locations: a data location that is the value of a + * single mapping in the block map tree, and the two locations of the block map pages and slots + * that are acquiring and releasing a reference to the location. The journal entry also stores an + * operation code that says whether the mapping is for a logical block or for the block map tree + * itself. + */ +struct recovery_journal_entry { + struct block_map_slot slot; + struct data_location mapping; + struct data_location unmapping; + enum journal_operation operation; +}; + +/* The packed, on-disk representation of a recovery journal entry. */ +struct packed_recovery_journal_entry { + /* + * In little-endian bit order: + * Bits 15..12: The four highest bits of the 36-bit physical block number of the block map + * tree page + * Bits 11..2: The 10-bit block map page slot number + * Bit 1..0: The journal_operation of the entry (this actually only requires 1 bit, but + * it is convenient to keep the extra bit as part of this field. + */ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + unsigned operation : 2; + unsigned slot_low : 6; + unsigned slot_high : 4; + unsigned pbn_high_nibble : 4; +#else + unsigned slot_low : 6; + unsigned operation : 2; + unsigned pbn_high_nibble : 4; + unsigned slot_high : 4; +#endif + + /* + * Bits 47..16: The 32 low-order bits of the block map page PBN, in little-endian byte + * order + */ + __le32 pbn_low_word; + + /* + * Bits 87..48: The five-byte block map entry encoding the location that will be stored in + * the block map page slot + */ + struct block_map_entry mapping; + + /* + * Bits 127..88: The five-byte block map entry encoding the location that was stored in the + * block map page slot + */ + struct block_map_entry unmapping; +} __packed; + +/* The packed, on-disk representation of an old format recovery journal entry. */ +struct packed_recovery_journal_entry_1 { + /* + * In little-endian bit order: + * Bits 15..12: The four highest bits of the 36-bit physical block number of the block map + * tree page + * Bits 11..2: The 10-bit block map page slot number + * Bits 1..0: The 2-bit journal_operation of the entry + * + */ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + unsigned operation : 2; + unsigned slot_low : 6; + unsigned slot_high : 4; + unsigned pbn_high_nibble : 4; +#else + unsigned slot_low : 6; + unsigned operation : 2; + unsigned pbn_high_nibble : 4; + unsigned slot_high : 4; +#endif + + /* + * Bits 47..16: The 32 low-order bits of the block map page PBN, in little-endian byte + * order + */ + __le32 pbn_low_word; + + /* + * Bits 87..48: The five-byte block map entry encoding the location that was or will be + * stored in the block map page slot + */ + struct block_map_entry block_map_entry; +} __packed; + +enum journal_operation_1 { + VDO_JOURNAL_DATA_DECREMENT = 0, + VDO_JOURNAL_DATA_INCREMENT = 1, + VDO_JOURNAL_BLOCK_MAP_DECREMENT = 2, + VDO_JOURNAL_BLOCK_MAP_INCREMENT = 3, +} __packed; + +struct recovery_block_header { + sequence_number_t block_map_head; /* Block map head sequence number */ + sequence_number_t slab_journal_head; /* Slab journal head seq. number */ + sequence_number_t sequence_number; /* Sequence number for this block */ + nonce_t nonce; /* A given VDO instance's nonce */ + block_count_t logical_blocks_used; /* Logical blocks in use */ + block_count_t block_map_data_blocks; /* Allocated block map pages */ + journal_entry_count_t entry_count; /* Number of entries written */ + u8 check_byte; /* The protection check byte */ + u8 recovery_count; /* Number of recoveries completed */ + enum vdo_metadata_type metadata_type; /* Metadata type */ +}; + +/* + * The packed, on-disk representation of a recovery journal block header. All fields are kept in + * little-endian byte order. + */ +struct packed_journal_header { + /* Block map head 64-bit sequence number */ + __le64 block_map_head; + + /* Slab journal head 64-bit sequence number */ + __le64 slab_journal_head; + + /* The 64-bit sequence number for this block */ + __le64 sequence_number; + + /* A given VDO instance's 64-bit nonce */ + __le64 nonce; + + /* 8-bit metadata type (should always be one for the recovery journal) */ + u8 metadata_type; + + /* 16-bit count of the entries encoded in the block */ + __le16 entry_count; + + /* 64-bit count of the logical blocks used when this block was opened */ + __le64 logical_blocks_used; + + /* 64-bit count of the block map blocks used when this block was opened */ + __le64 block_map_data_blocks; + + /* The protection check byte */ + u8 check_byte; + + /* The number of recoveries completed */ + u8 recovery_count; +} __packed; + +struct packed_journal_sector { + /* The protection check byte */ + u8 check_byte; + + /* The number of recoveries completed */ + u8 recovery_count; + + /* The number of entries in this sector */ + u8 entry_count; + + /* Journal entries for this sector */ + struct packed_recovery_journal_entry entries[]; +} __packed; + +enum { + /* The number of entries in each sector (except the last) when filled */ + RECOVERY_JOURNAL_ENTRIES_PER_SECTOR = + ((VDO_SECTOR_SIZE - sizeof(struct packed_journal_sector)) / + sizeof(struct packed_recovery_journal_entry)), + RECOVERY_JOURNAL_ENTRIES_PER_BLOCK = RECOVERY_JOURNAL_ENTRIES_PER_SECTOR * 7, + /* The number of entries in a v1 recovery journal block. */ + RECOVERY_JOURNAL_1_ENTRIES_PER_BLOCK = 311, + /* The number of entries in each v1 sector (except the last) when filled */ + RECOVERY_JOURNAL_1_ENTRIES_PER_SECTOR = + ((VDO_SECTOR_SIZE - sizeof(struct packed_journal_sector)) / + sizeof(struct packed_recovery_journal_entry_1)), + /* The number of entries in the last sector when a block is full */ + RECOVERY_JOURNAL_1_ENTRIES_IN_LAST_SECTOR = + (RECOVERY_JOURNAL_1_ENTRIES_PER_BLOCK % RECOVERY_JOURNAL_1_ENTRIES_PER_SECTOR), +}; + +/* A type representing a reference count of a block. */ +typedef u8 vdo_refcount_t; + +/* The absolute position of an entry in a recovery journal or slab journal. */ +struct journal_point { + sequence_number_t sequence_number; + journal_entry_count_t entry_count; +}; + +/* A packed, platform-independent encoding of a struct journal_point. */ +struct packed_journal_point { + /* + * The packed representation is the little-endian 64-bit representation of the low-order 48 + * bits of the sequence number, shifted up 16 bits, or'ed with the 16-bit entry count. + * + * Very long-term, the top 16 bits of the sequence number may not always be zero, as this + * encoding assumes--see BZ 1523240. + */ + __le64 encoded_point; +} __packed; + +/* Special vdo_refcount_t values. */ +#define EMPTY_REFERENCE_COUNT 0 +enum { + MAXIMUM_REFERENCE_COUNT = 254, + PROVISIONAL_REFERENCE_COUNT = 255, +}; + +enum { + COUNTS_PER_SECTOR = + ((VDO_SECTOR_SIZE - sizeof(struct packed_journal_point)) / sizeof(vdo_refcount_t)), + COUNTS_PER_BLOCK = COUNTS_PER_SECTOR * VDO_SECTORS_PER_BLOCK, +}; + +/* The format of each sector of a reference_block on disk. */ +struct packed_reference_sector { + struct packed_journal_point commit_point; + vdo_refcount_t counts[COUNTS_PER_SECTOR]; +} __packed; + +struct packed_reference_block { + struct packed_reference_sector sectors[VDO_SECTORS_PER_BLOCK]; +}; + +struct slab_depot_state_2_0 { + struct slab_config slab_config; + physical_block_number_t first_block; + physical_block_number_t last_block; + zone_count_t zone_count; +} __packed; + +extern const struct header VDO_SLAB_DEPOT_HEADER_2_0; + +/* + * vdo_slab journal blocks may have one of two formats, depending upon whether or not any of the + * entries in the block are block map increments. Since the steady state for a VDO is that all of + * the necessary block map pages will be allocated, most slab journal blocks will have only data + * entries. Such blocks can hold more entries, hence the two formats. + */ + +/* A single slab journal entry */ +struct slab_journal_entry { + slab_block_number sbn; + enum journal_operation operation; + bool increment; +}; + +/* A single slab journal entry in its on-disk form */ +typedef struct { + u8 offset_low8; + u8 offset_mid8; + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + unsigned offset_high7 : 7; + unsigned increment : 1; +#else + unsigned increment : 1; + unsigned offset_high7 : 7; +#endif +} __packed packed_slab_journal_entry; + +/* The unpacked representation of the header of a slab journal block */ +struct slab_journal_block_header { + /* Sequence number for head of journal */ + sequence_number_t head; + /* Sequence number for this block */ + sequence_number_t sequence_number; + /* The nonce for a given VDO instance */ + nonce_t nonce; + /* Recovery journal point for last entry */ + struct journal_point recovery_point; + /* Metadata type */ + enum vdo_metadata_type metadata_type; + /* Whether this block contains block map increments */ + bool has_block_map_increments; + /* The number of entries in the block */ + journal_entry_count_t entry_count; +}; + +/* + * The packed, on-disk representation of a slab journal block header. All fields are kept in + * little-endian byte order. + */ +struct packed_slab_journal_block_header { + /* 64-bit sequence number for head of journal */ + __le64 head; + /* 64-bit sequence number for this block */ + __le64 sequence_number; + /* Recovery journal point for the last entry, packed into 64 bits */ + struct packed_journal_point recovery_point; + /* The 64-bit nonce for a given VDO instance */ + __le64 nonce; + /* 8-bit metadata type (should always be two, for the slab journal) */ + u8 metadata_type; + /* Whether this block contains block map increments */ + bool has_block_map_increments; + /* 16-bit count of the entries encoded in the block */ + __le16 entry_count; +} __packed; + +enum { + VDO_SLAB_JOURNAL_PAYLOAD_SIZE = + VDO_BLOCK_SIZE - sizeof(struct packed_slab_journal_block_header), + VDO_SLAB_JOURNAL_FULL_ENTRIES_PER_BLOCK = (VDO_SLAB_JOURNAL_PAYLOAD_SIZE * 8) / 25, + VDO_SLAB_JOURNAL_ENTRY_TYPES_SIZE = + ((VDO_SLAB_JOURNAL_FULL_ENTRIES_PER_BLOCK - 1) / 8) + 1, + VDO_SLAB_JOURNAL_ENTRIES_PER_BLOCK = + (VDO_SLAB_JOURNAL_PAYLOAD_SIZE / sizeof(packed_slab_journal_entry)), +}; + +/* The payload of a slab journal block which has block map increments */ +struct full_slab_journal_entries { + /* The entries themselves */ + packed_slab_journal_entry entries[VDO_SLAB_JOURNAL_FULL_ENTRIES_PER_BLOCK]; + /* The bit map indicating which entries are block map increments */ + u8 entry_types[VDO_SLAB_JOURNAL_ENTRY_TYPES_SIZE]; +} __packed; + +typedef union { + /* Entries which include block map increments */ + struct full_slab_journal_entries full_entries; + /* Entries which are only data updates */ + packed_slab_journal_entry entries[VDO_SLAB_JOURNAL_ENTRIES_PER_BLOCK]; + /* Ensure the payload fills to the end of the block */ + u8 space[VDO_SLAB_JOURNAL_PAYLOAD_SIZE]; +} __packed slab_journal_payload; + +struct packed_slab_journal_block { + struct packed_slab_journal_block_header header; + slab_journal_payload payload; +} __packed; + +/* The offset of a slab journal tail block. */ +typedef u8 tail_block_offset_t; + +struct slab_summary_entry { + /* Bits 7..0: The offset of the tail block within the slab journal */ + tail_block_offset_t tail_block_offset; + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + /* Bits 13..8: A hint about the fullness of the slab */ + unsigned int fullness_hint : 6; + /* Bit 14: Whether the ref_counts must be loaded from the layer */ + unsigned int load_ref_counts : 1; + /* Bit 15: The believed cleanliness of this slab */ + unsigned int is_dirty : 1; +#else + /* Bit 15: The believed cleanliness of this slab */ + unsigned int is_dirty : 1; + /* Bit 14: Whether the ref_counts must be loaded from the layer */ + unsigned int load_ref_counts : 1; + /* Bits 13..8: A hint about the fullness of the slab */ + unsigned int fullness_hint : 6; +#endif +} __packed; + +enum { + VDO_SLAB_SUMMARY_FULLNESS_HINT_BITS = 6, + VDO_SLAB_SUMMARY_ENTRIES_PER_BLOCK = VDO_BLOCK_SIZE / sizeof(struct slab_summary_entry), + VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE = MAX_VDO_SLABS / VDO_SLAB_SUMMARY_ENTRIES_PER_BLOCK, + VDO_SLAB_SUMMARY_BLOCKS = VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE * MAX_VDO_PHYSICAL_ZONES, +}; + +struct layout { + physical_block_number_t start; + block_count_t size; + physical_block_number_t first_free; + physical_block_number_t last_free; + size_t num_partitions; + struct partition *head; +}; + +struct partition { + enum partition_id id; /* The id of this partition */ + physical_block_number_t offset; /* The offset into the layout of this partition */ + block_count_t count; /* The number of blocks in the partition */ + struct partition *next; /* A pointer to the next partition in the layout */ +}; + +struct layout_3_0 { + physical_block_number_t first_free; + physical_block_number_t last_free; + u8 partition_count; +} __packed; + +struct partition_3_0 { + enum partition_id id; + physical_block_number_t offset; + physical_block_number_t base; /* unused but retained for backwards compatibility */ + block_count_t count; +} __packed; + +/* + * The configuration of the VDO service. + */ +struct vdo_config { + block_count_t logical_blocks; /* number of logical blocks */ + block_count_t physical_blocks; /* number of physical blocks */ + block_count_t slab_size; /* number of blocks in a slab */ + block_count_t recovery_journal_size; /* number of recovery journal blocks */ + block_count_t slab_journal_blocks; /* number of slab journal blocks */ +}; + +/* This is the structure that captures the vdo fields saved as a super block component. */ +struct vdo_component { + enum vdo_state state; + u64 complete_recoveries; + u64 read_only_recoveries; + struct vdo_config config; + nonce_t nonce; +}; + +/* + * A packed, machine-independent, on-disk representation of the vdo_config in the VDO component + * data in the super block. + */ +struct packed_vdo_config { + __le64 logical_blocks; + __le64 physical_blocks; + __le64 slab_size; + __le64 recovery_journal_size; + __le64 slab_journal_blocks; +} __packed; + +/* + * A packed, machine-independent, on-disk representation of version 41.0 of the VDO component data + * in the super block. + */ +struct packed_vdo_component_41_0 { + __le32 state; + __le64 complete_recoveries; + __le64 read_only_recoveries; + struct packed_vdo_config config; + __le64 nonce; +} __packed; + +/* + * The version of the on-disk format of a VDO volume. This should be incremented any time the + * on-disk representation of any VDO structure changes. Changes which require only online upgrade + * steps should increment the minor version. Changes which require an offline upgrade or which can + * not be upgraded to at all should increment the major version and set the minor version to 0. + */ +extern const struct version_number VDO_VOLUME_VERSION_67_0; + +enum { + VDO_ENCODED_HEADER_SIZE = sizeof(struct packed_header), + BLOCK_MAP_COMPONENT_ENCODED_SIZE = + VDO_ENCODED_HEADER_SIZE + sizeof(struct block_map_state_2_0), + RECOVERY_JOURNAL_COMPONENT_ENCODED_SIZE = + VDO_ENCODED_HEADER_SIZE + sizeof(struct recovery_journal_state_7_0), + SLAB_DEPOT_COMPONENT_ENCODED_SIZE = + VDO_ENCODED_HEADER_SIZE + sizeof(struct slab_depot_state_2_0), + VDO_PARTITION_COUNT = 4, + VDO_LAYOUT_ENCODED_SIZE = (VDO_ENCODED_HEADER_SIZE + + sizeof(struct layout_3_0) + + (sizeof(struct partition_3_0) * VDO_PARTITION_COUNT)), + VDO_SUPER_BLOCK_FIXED_SIZE = VDO_ENCODED_HEADER_SIZE + sizeof(u32), + VDO_MAX_COMPONENT_DATA_SIZE = VDO_SECTOR_SIZE - VDO_SUPER_BLOCK_FIXED_SIZE, + VDO_COMPONENT_ENCODED_SIZE = + (sizeof(struct packed_version_number) + sizeof(struct packed_vdo_component_41_0)), + VDO_COMPONENT_DATA_OFFSET = VDO_ENCODED_HEADER_SIZE, + VDO_COMPONENT_DATA_SIZE = (sizeof(u32) + + sizeof(struct packed_version_number) + + VDO_COMPONENT_ENCODED_SIZE + + VDO_LAYOUT_ENCODED_SIZE + + RECOVERY_JOURNAL_COMPONENT_ENCODED_SIZE + + SLAB_DEPOT_COMPONENT_ENCODED_SIZE + + BLOCK_MAP_COMPONENT_ENCODED_SIZE), +}; + +/* The entirety of the component data encoded in the VDO super block. */ +struct vdo_component_states { + /* For backwards compatibility */ + u32 unused; + + /* The VDO volume version */ + struct version_number volume_version; + + /* Components */ + struct vdo_component vdo; + struct block_map_state_2_0 block_map; + struct recovery_journal_state_7_0 recovery_journal; + struct slab_depot_state_2_0 slab_depot; + + /* Our partitioning of the underlying storage */ + struct layout layout; +}; + +/** + * vdo_are_same_version() - Check whether two version numbers are the same. + * @version_a: The first version. + * @version_b: The second version. + * + * Return: true if the two versions are the same. + */ +static inline bool vdo_are_same_version(struct version_number version_a, + struct version_number version_b) +{ + return ((version_a.major_version == version_b.major_version) && + (version_a.minor_version == version_b.minor_version)); +} + +/** + * vdo_is_upgradable_version() - Check whether an actual version is upgradable to an expected + * version. + * @expected_version: The expected version. + * @actual_version: The version being validated. + * + * An actual version is upgradable if its major number is expected but its minor number differs, + * and the expected version's minor number is greater than the actual version's minor number. + * + * Return: true if the actual version is upgradable. + */ +static inline bool vdo_is_upgradable_version(struct version_number expected_version, + struct version_number actual_version) +{ + return ((expected_version.major_version == actual_version.major_version) && + (expected_version.minor_version > actual_version.minor_version)); +} + +int __must_check vdo_validate_header(const struct header *expected_header, + const struct header *actual_header, bool exact_size, + const char *component_name); + +void vdo_encode_header(u8 *buffer, size_t *offset, const struct header *header); +void vdo_decode_header(u8 *buffer, size_t *offset, struct header *header); + +/** + * vdo_pack_version_number() - Convert a version_number to its packed on-disk representation. + * @version: The version number to convert. + * + * Return: the platform-independent representation of the version + */ +static inline struct packed_version_number vdo_pack_version_number(struct version_number version) +{ + return (struct packed_version_number) { + .major_version = __cpu_to_le32(version.major_version), + .minor_version = __cpu_to_le32(version.minor_version), + }; +} + +/** + * vdo_unpack_version_number() - Convert a packed_version_number to its native in-memory + * representation. + * @version: The version number to convert. + * + * Return: The platform-independent representation of the version. + */ +static inline struct version_number vdo_unpack_version_number(struct packed_version_number version) +{ + return (struct version_number) { + .major_version = __le32_to_cpu(version.major_version), + .minor_version = __le32_to_cpu(version.minor_version), + }; +} + +/** + * vdo_pack_header() - Convert a component header to its packed on-disk representation. + * @header: The header to convert. + * + * Return: the platform-independent representation of the header + */ +static inline struct packed_header vdo_pack_header(const struct header *header) +{ + return (struct packed_header) { + .id = __cpu_to_le32(header->id), + .version = vdo_pack_version_number(header->version), + .size = __cpu_to_le64(header->size), + }; +} + +/** + * vdo_unpack_header() - Convert a packed_header to its native in-memory representation. + * @header: The header to convert. + * + * Return: The platform-independent representation of the version. + */ +static inline struct header vdo_unpack_header(const struct packed_header *header) +{ + return (struct header) { + .id = __le32_to_cpu(header->id), + .version = vdo_unpack_version_number(header->version), + .size = __le64_to_cpu(header->size), + }; +} + +/** + * vdo_get_index_region_start() - Get the start of the index region from a geometry. + * @geometry: The geometry. + * + * Return: The start of the index region. + */ +static inline physical_block_number_t __must_check +vdo_get_index_region_start(struct volume_geometry geometry) +{ + return geometry.regions[VDO_INDEX_REGION].start_block; +} + +/** + * vdo_get_data_region_start() - Get the start of the data region from a geometry. + * @geometry: The geometry. + * + * Return: The start of the data region. + */ +static inline physical_block_number_t __must_check +vdo_get_data_region_start(struct volume_geometry geometry) +{ + return geometry.regions[VDO_DATA_REGION].start_block; +} + +/** + * vdo_get_index_region_size() - Get the size of the index region from a geometry. + * @geometry: The geometry. + * + * Return: The size of the index region. + */ +static inline physical_block_number_t __must_check +vdo_get_index_region_size(struct volume_geometry geometry) +{ + return vdo_get_data_region_start(geometry) - + vdo_get_index_region_start(geometry); +} + +int __must_check vdo_parse_geometry_block(unsigned char *block, + struct volume_geometry *geometry); + +static inline bool vdo_is_state_compressed(const enum block_mapping_state mapping_state) +{ + return (mapping_state > VDO_MAPPING_STATE_UNCOMPRESSED); +} + +static inline struct block_map_entry +vdo_pack_block_map_entry(physical_block_number_t pbn, enum block_mapping_state mapping_state) +{ + return (struct block_map_entry) { + .mapping_state = (mapping_state & 0x0F), + .pbn_high_nibble = ((pbn >> 32) & 0x0F), + .pbn_low_word = __cpu_to_le32(pbn & UINT_MAX), + }; +} + +static inline struct data_location vdo_unpack_block_map_entry(const struct block_map_entry *entry) +{ + physical_block_number_t low32 = __le32_to_cpu(entry->pbn_low_word); + physical_block_number_t high4 = entry->pbn_high_nibble; + + return (struct data_location) { + .pbn = ((high4 << 32) | low32), + .state = entry->mapping_state, + }; +} + +static inline bool vdo_is_mapped_location(const struct data_location *location) +{ + return (location->state != VDO_MAPPING_STATE_UNMAPPED); +} + +static inline bool vdo_is_valid_location(const struct data_location *location) +{ + if (location->pbn == VDO_ZERO_BLOCK) + return !vdo_is_state_compressed(location->state); + else + return vdo_is_mapped_location(location); +} + +static inline physical_block_number_t __must_check +vdo_get_block_map_page_pbn(const struct block_map_page *page) +{ + return __le64_to_cpu(page->header.pbn); +} + +struct block_map_page *vdo_format_block_map_page(void *buffer, nonce_t nonce, + physical_block_number_t pbn, + bool initialized); + +enum block_map_page_validity __must_check vdo_validate_block_map_page(struct block_map_page *page, + nonce_t nonce, + physical_block_number_t pbn); + +static inline page_count_t vdo_compute_block_map_page_count(block_count_t entries) +{ + return DIV_ROUND_UP(entries, VDO_BLOCK_MAP_ENTRIES_PER_PAGE); +} + +block_count_t __must_check vdo_compute_new_forest_pages(root_count_t root_count, + struct boundary *old_sizes, + block_count_t entries, + struct boundary *new_sizes); + +/** + * vdo_pack_recovery_journal_entry() - Return the packed, on-disk representation of a recovery + * journal entry. + * @entry: The journal entry to pack. + * + * Return: The packed representation of the journal entry. + */ +static inline struct packed_recovery_journal_entry +vdo_pack_recovery_journal_entry(const struct recovery_journal_entry *entry) +{ + return (struct packed_recovery_journal_entry) { + .operation = entry->operation, + .slot_low = entry->slot.slot & 0x3F, + .slot_high = (entry->slot.slot >> 6) & 0x0F, + .pbn_high_nibble = (entry->slot.pbn >> 32) & 0x0F, + .pbn_low_word = __cpu_to_le32(entry->slot.pbn & UINT_MAX), + .mapping = vdo_pack_block_map_entry(entry->mapping.pbn, + entry->mapping.state), + .unmapping = vdo_pack_block_map_entry(entry->unmapping.pbn, + entry->unmapping.state), + }; +} + +/** + * vdo_unpack_recovery_journal_entry() - Unpack the on-disk representation of a recovery journal + * entry. + * @entry: The recovery journal entry to unpack. + * + * Return: The unpacked entry. + */ +static inline struct recovery_journal_entry +vdo_unpack_recovery_journal_entry(const struct packed_recovery_journal_entry *entry) +{ + physical_block_number_t low32 = __le32_to_cpu(entry->pbn_low_word); + physical_block_number_t high4 = entry->pbn_high_nibble; + + return (struct recovery_journal_entry) { + .operation = entry->operation, + .slot = { + .pbn = ((high4 << 32) | low32), + .slot = (entry->slot_low | (entry->slot_high << 6)), + }, + .mapping = vdo_unpack_block_map_entry(&entry->mapping), + .unmapping = vdo_unpack_block_map_entry(&entry->unmapping), + }; +} + +const char * __must_check vdo_get_journal_operation_name(enum journal_operation operation); + +/** + * vdo_is_valid_recovery_journal_sector() - Determine whether the header of the given sector could + * describe a valid sector for the given journal block + * header. + * @header: The unpacked block header to compare against. + * @sector: The packed sector to check. + * @sector_number: The number of the sector being checked. + * + * Return: true if the sector matches the block header. + */ +static inline bool __must_check +vdo_is_valid_recovery_journal_sector(const struct recovery_block_header *header, + const struct packed_journal_sector *sector, + u8 sector_number) +{ + if ((header->check_byte != sector->check_byte) || + (header->recovery_count != sector->recovery_count)) + return false; + + if (header->metadata_type == VDO_METADATA_RECOVERY_JOURNAL_2) + return sector->entry_count <= RECOVERY_JOURNAL_ENTRIES_PER_SECTOR; + + if (sector_number == 7) + return sector->entry_count <= RECOVERY_JOURNAL_1_ENTRIES_IN_LAST_SECTOR; + + return sector->entry_count <= RECOVERY_JOURNAL_1_ENTRIES_PER_SECTOR; +} + +/** + * vdo_compute_recovery_journal_block_number() - Compute the physical block number of the recovery + * journal block which would have a given sequence + * number. + * @journal_size: The size of the journal. + * @sequence_number: The sequence number. + * + * Return: The pbn of the journal block which would the specified sequence number. + */ +static inline physical_block_number_t __must_check +vdo_compute_recovery_journal_block_number(block_count_t journal_size, + sequence_number_t sequence_number) +{ + /* + * Since journal size is a power of two, the block number modulus can just be extracted + * from the low-order bits of the sequence. + */ + return (sequence_number & (journal_size - 1)); +} + +/** + * vdo_get_journal_block_sector() - Find the recovery journal sector from the block header and + * sector number. + * @header: The header of the recovery journal block. + * @sector_number: The index of the sector (1-based). + * + * Return: A packed recovery journal sector. + */ +static inline struct packed_journal_sector * __must_check +vdo_get_journal_block_sector(struct packed_journal_header *header, int sector_number) +{ + char *sector_data = ((char *) header) + (VDO_SECTOR_SIZE * sector_number); + + return (struct packed_journal_sector *) sector_data; +} + +/** + * vdo_pack_recovery_block_header() - Generate the packed representation of a recovery block + * header. + * @header: The header containing the values to encode. + * @packed: The header into which to pack the values. + */ +static inline void vdo_pack_recovery_block_header(const struct recovery_block_header *header, + struct packed_journal_header *packed) +{ + *packed = (struct packed_journal_header) { + .block_map_head = __cpu_to_le64(header->block_map_head), + .slab_journal_head = __cpu_to_le64(header->slab_journal_head), + .sequence_number = __cpu_to_le64(header->sequence_number), + .nonce = __cpu_to_le64(header->nonce), + .logical_blocks_used = __cpu_to_le64(header->logical_blocks_used), + .block_map_data_blocks = __cpu_to_le64(header->block_map_data_blocks), + .entry_count = __cpu_to_le16(header->entry_count), + .check_byte = header->check_byte, + .recovery_count = header->recovery_count, + .metadata_type = header->metadata_type, + }; +} + +/** + * vdo_unpack_recovery_block_header() - Decode the packed representation of a recovery block + * header. + * @packed: The packed header to decode. + * + * Return: The unpacked header. + */ +static inline struct recovery_block_header +vdo_unpack_recovery_block_header(const struct packed_journal_header *packed) +{ + return (struct recovery_block_header) { + .block_map_head = __le64_to_cpu(packed->block_map_head), + .slab_journal_head = __le64_to_cpu(packed->slab_journal_head), + .sequence_number = __le64_to_cpu(packed->sequence_number), + .nonce = __le64_to_cpu(packed->nonce), + .logical_blocks_used = __le64_to_cpu(packed->logical_blocks_used), + .block_map_data_blocks = __le64_to_cpu(packed->block_map_data_blocks), + .entry_count = __le16_to_cpu(packed->entry_count), + .check_byte = packed->check_byte, + .recovery_count = packed->recovery_count, + .metadata_type = packed->metadata_type, + }; +} + +/** + * vdo_compute_slab_count() - Compute the number of slabs a depot with given parameters would have. + * @first_block: PBN of the first data block. + * @last_block: PBN of the last data block. + * @slab_size_shift: Exponent for the number of blocks per slab. + * + * Return: The number of slabs. + */ +static inline slab_count_t vdo_compute_slab_count(physical_block_number_t first_block, + physical_block_number_t last_block, + unsigned int slab_size_shift) +{ + return (slab_count_t) ((last_block - first_block) >> slab_size_shift); +} + +int __must_check vdo_configure_slab_depot(const struct partition *partition, + struct slab_config slab_config, + zone_count_t zone_count, + struct slab_depot_state_2_0 *state); + +int __must_check vdo_configure_slab(block_count_t slab_size, + block_count_t slab_journal_blocks, + struct slab_config *slab_config); + +/** + * vdo_get_saved_reference_count_size() - Get the number of blocks required to save a reference + * counts state covering the specified number of data + * blocks. + * @block_count: The number of physical data blocks that can be referenced. + * + * Return: The number of blocks required to save reference counts with the given block count. + */ +static inline block_count_t vdo_get_saved_reference_count_size(block_count_t block_count) +{ + return DIV_ROUND_UP(block_count, COUNTS_PER_BLOCK); +} + +/** + * vdo_get_slab_journal_start_block() - Get the physical block number of the start of the slab + * journal relative to the start block allocator partition. + * @slab_config: The slab configuration of the VDO. + * @origin: The first block of the slab. + */ +static inline physical_block_number_t __must_check +vdo_get_slab_journal_start_block(const struct slab_config *slab_config, + physical_block_number_t origin) +{ + return origin + slab_config->data_blocks + slab_config->reference_count_blocks; +} + +/** + * vdo_advance_journal_point() - Move the given journal point forward by one entry. + * @point: The journal point to adjust. + * @entries_per_block: The number of entries in one full block. + */ +static inline void vdo_advance_journal_point(struct journal_point *point, + journal_entry_count_t entries_per_block) +{ + point->entry_count++; + if (point->entry_count == entries_per_block) { + point->sequence_number++; + point->entry_count = 0; + } +} + +/** + * vdo_before_journal_point() - Check whether the first point precedes the second point. + * @first: The first journal point. + * @second: The second journal point. + * + * Return: true if the first point precedes the second point. + */ +static inline bool vdo_before_journal_point(const struct journal_point *first, + const struct journal_point *second) +{ + return ((first->sequence_number < second->sequence_number) || + ((first->sequence_number == second->sequence_number) && + (first->entry_count < second->entry_count))); +} + +/** + * vdo_pack_journal_point() - Encode the journal location represented by a + * journal_point into a packed_journal_point. + * @unpacked: The unpacked input point. + * @packed: The packed output point. + */ +static inline void vdo_pack_journal_point(const struct journal_point *unpacked, + struct packed_journal_point *packed) +{ + packed->encoded_point = + __cpu_to_le64((unpacked->sequence_number << 16) | unpacked->entry_count); +} + +/** + * vdo_unpack_journal_point() - Decode the journal location represented by a packed_journal_point + * into a journal_point. + * @packed: The packed input point. + * @unpacked: The unpacked output point. + */ +static inline void vdo_unpack_journal_point(const struct packed_journal_point *packed, + struct journal_point *unpacked) +{ + u64 native = __le64_to_cpu(packed->encoded_point); + + unpacked->sequence_number = (native >> 16); + unpacked->entry_count = (native & 0xffff); +} + +/** + * vdo_pack_slab_journal_block_header() - Generate the packed representation of a slab block + * header. + * @header: The header containing the values to encode. + * @packed: The header into which to pack the values. + */ +static inline void +vdo_pack_slab_journal_block_header(const struct slab_journal_block_header *header, + struct packed_slab_journal_block_header *packed) +{ + packed->head = __cpu_to_le64(header->head); + packed->sequence_number = __cpu_to_le64(header->sequence_number); + packed->nonce = __cpu_to_le64(header->nonce); + packed->entry_count = __cpu_to_le16(header->entry_count); + packed->metadata_type = header->metadata_type; + packed->has_block_map_increments = header->has_block_map_increments; + + vdo_pack_journal_point(&header->recovery_point, &packed->recovery_point); +} + +/** + * vdo_unpack_slab_journal_block_header() - Decode the packed representation of a slab block + * header. + * @packed: The packed header to decode. + * @header: The header into which to unpack the values. + */ +static inline void +vdo_unpack_slab_journal_block_header(const struct packed_slab_journal_block_header *packed, + struct slab_journal_block_header *header) +{ + *header = (struct slab_journal_block_header) { + .head = __le64_to_cpu(packed->head), + .sequence_number = __le64_to_cpu(packed->sequence_number), + .nonce = __le64_to_cpu(packed->nonce), + .entry_count = __le16_to_cpu(packed->entry_count), + .metadata_type = packed->metadata_type, + .has_block_map_increments = packed->has_block_map_increments, + }; + vdo_unpack_journal_point(&packed->recovery_point, &header->recovery_point); +} + +/** + * vdo_pack_slab_journal_entry() - Generate the packed encoding of a slab journal entry. + * @packed: The entry into which to pack the values. + * @sbn: The slab block number of the entry to encode. + * @is_increment: The increment flag. + */ +static inline void vdo_pack_slab_journal_entry(packed_slab_journal_entry *packed, + slab_block_number sbn, bool is_increment) +{ + packed->offset_low8 = (sbn & 0x0000FF); + packed->offset_mid8 = (sbn & 0x00FF00) >> 8; + packed->offset_high7 = (sbn & 0x7F0000) >> 16; + packed->increment = is_increment ? 1 : 0; +} + +/** + * vdo_unpack_slab_journal_entry() - Decode the packed representation of a slab journal entry. + * @packed: The packed entry to decode. + * + * Return: The decoded slab journal entry. + */ +static inline struct slab_journal_entry __must_check +vdo_unpack_slab_journal_entry(const packed_slab_journal_entry *packed) +{ + struct slab_journal_entry entry; + + entry.sbn = packed->offset_high7; + entry.sbn <<= 8; + entry.sbn |= packed->offset_mid8; + entry.sbn <<= 8; + entry.sbn |= packed->offset_low8; + entry.operation = VDO_JOURNAL_DATA_REMAPPING; + entry.increment = packed->increment; + return entry; +} + +struct slab_journal_entry __must_check +vdo_decode_slab_journal_entry(struct packed_slab_journal_block *block, + journal_entry_count_t entry_count); + +/** + * vdo_get_slab_summary_hint_shift() - Compute the shift for slab summary hints. + * @slab_size_shift: Exponent for the number of blocks per slab. + * + * Return: The hint shift. + */ +static inline u8 __must_check vdo_get_slab_summary_hint_shift(unsigned int slab_size_shift) +{ + return ((slab_size_shift > VDO_SLAB_SUMMARY_FULLNESS_HINT_BITS) ? + (slab_size_shift - VDO_SLAB_SUMMARY_FULLNESS_HINT_BITS) : + 0); +} + +int __must_check vdo_initialize_layout(block_count_t size, + physical_block_number_t offset, + block_count_t block_map_blocks, + block_count_t journal_blocks, + block_count_t summary_blocks, + struct layout *layout); + +void vdo_uninitialize_layout(struct layout *layout); + +int __must_check vdo_get_partition(struct layout *layout, enum partition_id id, + struct partition **partition_ptr); + +struct partition * __must_check vdo_get_known_partition(struct layout *layout, + enum partition_id id); + +int vdo_validate_config(const struct vdo_config *config, + block_count_t physical_block_count, + block_count_t logical_block_count); + +void vdo_destroy_component_states(struct vdo_component_states *states); + +int __must_check vdo_decode_component_states(u8 *buffer, + struct volume_geometry *geometry, + struct vdo_component_states *states); + +int __must_check vdo_validate_component_states(struct vdo_component_states *states, + nonce_t geometry_nonce, + block_count_t physical_size, + block_count_t logical_size); + +void vdo_encode_super_block(u8 *buffer, struct vdo_component_states *states); +int __must_check vdo_decode_super_block(u8 *buffer); + +/* We start with 0L and postcondition with ~0L to match our historical usage in userspace. */ +static inline u32 vdo_crc32(const void *buf, unsigned long len) +{ + return (crc32(0L, buf, len) ^ ~0L); +} + +#endif /* VDO_ENCODINGS_H */ diff --git a/drivers/md/dm-vdo/errors.c b/drivers/md/dm-vdo/errors.c new file mode 100644 index 000000000000..6f89eb1c63a3 --- /dev/null +++ b/drivers/md/dm-vdo/errors.c @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "errors.h" + +#include +#include + +#include "logger.h" +#include "permassert.h" +#include "string-utils.h" + +static const struct error_info successful = { "UDS_SUCCESS", "Success" }; + +static const char *const message_table[] = { + [EPERM] = "Operation not permitted", + [ENOENT] = "No such file or directory", + [ESRCH] = "No such process", + [EINTR] = "Interrupted system call", + [EIO] = "Input/output error", + [ENXIO] = "No such device or address", + [E2BIG] = "Argument list too long", + [ENOEXEC] = "Exec format error", + [EBADF] = "Bad file descriptor", + [ECHILD] = "No child processes", + [EAGAIN] = "Resource temporarily unavailable", + [ENOMEM] = "Cannot allocate memory", + [EACCES] = "Permission denied", + [EFAULT] = "Bad address", + [ENOTBLK] = "Block device required", + [EBUSY] = "Device or resource busy", + [EEXIST] = "File exists", + [EXDEV] = "Invalid cross-device link", + [ENODEV] = "No such device", + [ENOTDIR] = "Not a directory", + [EISDIR] = "Is a directory", + [EINVAL] = "Invalid argument", + [ENFILE] = "Too many open files in system", + [EMFILE] = "Too many open files", + [ENOTTY] = "Inappropriate ioctl for device", + [ETXTBSY] = "Text file busy", + [EFBIG] = "File too large", + [ENOSPC] = "No space left on device", + [ESPIPE] = "Illegal seek", + [EROFS] = "Read-only file system", + [EMLINK] = "Too many links", + [EPIPE] = "Broken pipe", + [EDOM] = "Numerical argument out of domain", + [ERANGE] = "Numerical result out of range" +}; + +static const struct error_info error_list[] = { + { "UDS_OVERFLOW", "Index overflow" }, + { "UDS_INVALID_ARGUMENT", "Invalid argument passed to internal routine" }, + { "UDS_BAD_STATE", "UDS data structures are in an invalid state" }, + { "UDS_DUPLICATE_NAME", "Attempt to enter the same name into a delta index twice" }, + { "UDS_ASSERTION_FAILED", "Assertion failed" }, + { "UDS_QUEUED", "Request queued" }, + { "UDS_ALREADY_REGISTERED", "Error range already registered" }, + { "UDS_OUT_OF_RANGE", "Cannot access data outside specified limits" }, + { "UDS_DISABLED", "UDS library context is disabled" }, + { "UDS_UNSUPPORTED_VERSION", "Unsupported version" }, + { "UDS_CORRUPT_DATA", "Some index structure is corrupt" }, + { "UDS_NO_INDEX", "No index found" }, + { "UDS_INDEX_NOT_SAVED_CLEANLY", "Index not saved cleanly" }, +}; + +struct error_block { + const char *name; + int base; + int last; + int max; + const struct error_info *infos; +}; + +#define MAX_ERROR_BLOCKS 6 + +static struct { + int allocated; + int count; + struct error_block blocks[MAX_ERROR_BLOCKS]; +} registered_errors = { + .allocated = MAX_ERROR_BLOCKS, + .count = 1, + .blocks = { { + .name = "UDS Error", + .base = UDS_ERROR_CODE_BASE, + .last = UDS_ERROR_CODE_LAST, + .max = UDS_ERROR_CODE_BLOCK_END, + .infos = error_list, + } }, +}; + +/* Get the error info for an error number. Also returns the name of the error block, if known. */ +static const char *get_error_info(int errnum, const struct error_info **info_ptr) +{ + struct error_block *block; + + if (errnum == UDS_SUCCESS) { + *info_ptr = &successful; + return NULL; + } + + for (block = registered_errors.blocks; + block < registered_errors.blocks + registered_errors.count; + block++) { + if ((errnum >= block->base) && (errnum < block->last)) { + *info_ptr = block->infos + (errnum - block->base); + return block->name; + } else if ((errnum >= block->last) && (errnum < block->max)) { + *info_ptr = NULL; + return block->name; + } + } + + return NULL; +} + +/* Return a string describing a system error message. */ +static const char *system_string_error(int errnum, char *buf, size_t buflen) +{ + size_t len; + const char *error_string = NULL; + + if ((errnum > 0) && (errnum < ARRAY_SIZE(message_table))) + error_string = message_table[errnum]; + + len = ((error_string == NULL) ? + snprintf(buf, buflen, "Unknown error %d", errnum) : + snprintf(buf, buflen, "%s", error_string)); + if (len < buflen) + return buf; + + buf[0] = '\0'; + return "System error"; +} + +/* Convert an error code to a descriptive string. */ +const char *uds_string_error(int errnum, char *buf, size_t buflen) +{ + char *buffer = buf; + char *buf_end = buf + buflen; + const struct error_info *info = NULL; + const char *block_name; + + if (buf == NULL) + return NULL; + + if (errnum < 0) + errnum = -errnum; + + block_name = get_error_info(errnum, &info); + if (block_name != NULL) { + if (info != NULL) { + buffer = vdo_append_to_buffer(buffer, buf_end, "%s: %s", + block_name, info->message); + } else { + buffer = vdo_append_to_buffer(buffer, buf_end, "Unknown %s %d", + block_name, errnum); + } + } else if (info != NULL) { + buffer = vdo_append_to_buffer(buffer, buf_end, "%s", info->message); + } else { + const char *tmp = system_string_error(errnum, buffer, buf_end - buffer); + + if (tmp != buffer) + buffer = vdo_append_to_buffer(buffer, buf_end, "%s", tmp); + else + buffer += strlen(tmp); + } + + return buf; +} + +/* Convert an error code to its name. */ +const char *uds_string_error_name(int errnum, char *buf, size_t buflen) +{ + char *buffer = buf; + char *buf_end = buf + buflen; + const struct error_info *info = NULL; + const char *block_name; + + if (errnum < 0) + errnum = -errnum; + + block_name = get_error_info(errnum, &info); + if (block_name != NULL) { + if (info != NULL) { + buffer = vdo_append_to_buffer(buffer, buf_end, "%s", info->name); + } else { + buffer = vdo_append_to_buffer(buffer, buf_end, "%s %d", + block_name, errnum); + } + } else if (info != NULL) { + buffer = vdo_append_to_buffer(buffer, buf_end, "%s", info->name); + } else { + const char *tmp; + + tmp = system_string_error(errnum, buffer, buf_end - buffer); + if (tmp != buffer) + buffer = vdo_append_to_buffer(buffer, buf_end, "%s", tmp); + else + buffer += strlen(tmp); + } + + return buf; +} + +/* + * Translate an error code into a value acceptable to the kernel. The input error code may be a + * system-generated value (such as -EIO), or an internal UDS status code. The result will be a + * negative errno value. + */ +int uds_status_to_errno(int error) +{ + char error_name[VDO_MAX_ERROR_NAME_SIZE]; + char error_message[VDO_MAX_ERROR_MESSAGE_SIZE]; + + /* 0 is success, and negative values are already system error codes. */ + if (likely(error <= 0)) + return error; + + if (error < 1024) { + /* This is probably an errno from userspace. */ + return -error; + } + + /* Internal UDS errors */ + switch (error) { + case UDS_NO_INDEX: + case UDS_CORRUPT_DATA: + /* The index doesn't exist or can't be recovered. */ + return -ENOENT; + + case UDS_INDEX_NOT_SAVED_CLEANLY: + case UDS_UNSUPPORTED_VERSION: + /* + * The index exists, but can't be loaded. Tell the client it exists so they don't + * destroy it inadvertently. + */ + return -EEXIST; + + case UDS_DISABLED: + /* The session is unusable; only returned by requests. */ + return -EIO; + + default: + /* Translate an unexpected error into something generic. */ + vdo_log_info("%s: mapping status code %d (%s: %s) to -EIO", + __func__, error, + uds_string_error_name(error, error_name, + sizeof(error_name)), + uds_string_error(error, error_message, + sizeof(error_message))); + return -EIO; + } +} + +/* + * Register a block of error codes. + * + * @block_name: the name of the block of error codes + * @first_error: the first error code in the block + * @next_free_error: one past the highest possible error in the block + * @infos: a pointer to the error info array for the block + * @info_size: the size of the error info array + */ +int uds_register_error_block(const char *block_name, int first_error, + int next_free_error, const struct error_info *infos, + size_t info_size) +{ + int result; + struct error_block *block; + struct error_block new_block = { + .name = block_name, + .base = first_error, + .last = first_error + (info_size / sizeof(struct error_info)), + .max = next_free_error, + .infos = infos, + }; + + result = VDO_ASSERT(first_error < next_free_error, + "well-defined error block range"); + if (result != VDO_SUCCESS) + return result; + + if (registered_errors.count == registered_errors.allocated) { + /* This should never happen. */ + return UDS_OVERFLOW; + } + + for (block = registered_errors.blocks; + block < registered_errors.blocks + registered_errors.count; + block++) { + if (strcmp(block_name, block->name) == 0) + return UDS_DUPLICATE_NAME; + + /* Ensure error ranges do not overlap. */ + if ((first_error < block->max) && (next_free_error > block->base)) + return UDS_ALREADY_REGISTERED; + } + + registered_errors.blocks[registered_errors.count++] = new_block; + return UDS_SUCCESS; +} diff --git a/drivers/md/dm-vdo/errors.h b/drivers/md/dm-vdo/errors.h new file mode 100644 index 000000000000..24e0e745fd5f --- /dev/null +++ b/drivers/md/dm-vdo/errors.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_ERRORS_H +#define UDS_ERRORS_H + +#include +#include + +/* Custom error codes and error-related utilities */ +#define VDO_SUCCESS 0 + +/* Valid status codes for internal UDS functions. */ +enum uds_status_codes { + /* Successful return */ + UDS_SUCCESS = VDO_SUCCESS, + /* Used as a base value for reporting internal errors */ + UDS_ERROR_CODE_BASE = 1024, + /* Index overflow */ + UDS_OVERFLOW = UDS_ERROR_CODE_BASE, + /* Invalid argument passed to internal routine */ + UDS_INVALID_ARGUMENT, + /* UDS data structures are in an invalid state */ + UDS_BAD_STATE, + /* Attempt to enter the same name into an internal structure twice */ + UDS_DUPLICATE_NAME, + /* An assertion failed */ + UDS_ASSERTION_FAILED, + /* A request has been queued for later processing (not an error) */ + UDS_QUEUED, + /* This error range has already been registered */ + UDS_ALREADY_REGISTERED, + /* Attempt to read or write data outside the valid range */ + UDS_OUT_OF_RANGE, + /* The index session is disabled */ + UDS_DISABLED, + /* The index configuration or volume format is no longer supported */ + UDS_UNSUPPORTED_VERSION, + /* Some index structure is corrupt */ + UDS_CORRUPT_DATA, + /* No index state found */ + UDS_NO_INDEX, + /* Attempt to access incomplete index save data */ + UDS_INDEX_NOT_SAVED_CLEANLY, + /* One more than the last UDS_INTERNAL error code */ + UDS_ERROR_CODE_LAST, + /* One more than the last error this block will ever use */ + UDS_ERROR_CODE_BLOCK_END = UDS_ERROR_CODE_BASE + 440, +}; + +enum { + VDO_MAX_ERROR_NAME_SIZE = 80, + VDO_MAX_ERROR_MESSAGE_SIZE = 128, +}; + +struct error_info { + const char *name; + const char *message; +}; + +const char * __must_check uds_string_error(int errnum, char *buf, size_t buflen); + +const char *uds_string_error_name(int errnum, char *buf, size_t buflen); + +int uds_status_to_errno(int error); + +int uds_register_error_block(const char *block_name, int first_error, + int last_reserved_error, const struct error_info *infos, + size_t info_size); + +#endif /* UDS_ERRORS_H */ diff --git a/drivers/md/dm-vdo/flush.c b/drivers/md/dm-vdo/flush.c new file mode 100644 index 000000000000..57e87f0d7069 --- /dev/null +++ b/drivers/md/dm-vdo/flush.c @@ -0,0 +1,560 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "flush.h" + +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "admin-state.h" +#include "completion.h" +#include "io-submitter.h" +#include "logical-zone.h" +#include "slab-depot.h" +#include "types.h" +#include "vdo.h" + +struct flusher { + struct vdo_completion completion; + /* The vdo to which this flusher belongs */ + struct vdo *vdo; + /* The administrative state of the flusher */ + struct admin_state state; + /* The current flush generation of the vdo */ + sequence_number_t flush_generation; + /* The first unacknowledged flush generation */ + sequence_number_t first_unacknowledged_generation; + /* The queue of flush requests waiting to notify other threads */ + struct vdo_wait_queue notifiers; + /* The queue of flush requests waiting for VIOs to complete */ + struct vdo_wait_queue pending_flushes; + /* The flush generation for which notifications are being sent */ + sequence_number_t notify_generation; + /* The logical zone to notify next */ + struct logical_zone *logical_zone_to_notify; + /* The ID of the thread on which flush requests should be made */ + thread_id_t thread_id; + /* The pool of flush requests */ + mempool_t *flush_pool; + /* Bios waiting for a flush request to become available */ + struct bio_list waiting_flush_bios; + /* The lock to protect the previous fields */ + spinlock_t lock; + /* The rotor for selecting the bio queue for submitting flush bios */ + zone_count_t bio_queue_rotor; + /* The number of flushes submitted to the current bio queue */ + int flush_count; +}; + +/** + * assert_on_flusher_thread() - Check that we are on the flusher thread. + * @flusher: The flusher. + * @caller: The function which is asserting. + */ +static inline void assert_on_flusher_thread(struct flusher *flusher, const char *caller) +{ + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == flusher->thread_id), + "%s() called from flusher thread", caller); +} + +/** + * as_flusher() - Convert a generic vdo_completion to a flusher. + * @completion: The completion to convert. + * + * Return: The completion as a flusher. + */ +static struct flusher *as_flusher(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_FLUSH_NOTIFICATION_COMPLETION); + return container_of(completion, struct flusher, completion); +} + +/** + * completion_as_vdo_flush() - Convert a generic vdo_completion to a vdo_flush. + * @completion: The completion to convert. + * + * Return: The completion as a vdo_flush. + */ +static inline struct vdo_flush *completion_as_vdo_flush(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_FLUSH_COMPLETION); + return container_of(completion, struct vdo_flush, completion); +} + +/** + * vdo_waiter_as_flush() - Convert a vdo_flush's generic wait queue entry back to the vdo_flush. + * @waiter: The wait queue entry to convert. + * + * Return: The wait queue entry as a vdo_flush. + */ +static struct vdo_flush *vdo_waiter_as_flush(struct vdo_waiter *waiter) +{ + return container_of(waiter, struct vdo_flush, waiter); +} + +static void *allocate_flush(gfp_t gfp_mask, void *pool_data) +{ + struct vdo_flush *flush = NULL; + + if ((gfp_mask & GFP_NOWAIT) == GFP_NOWAIT) { + flush = vdo_allocate_memory_nowait(sizeof(struct vdo_flush), __func__); + } else { + int result = vdo_allocate(1, struct vdo_flush, __func__, &flush); + + if (result != VDO_SUCCESS) + vdo_log_error_strerror(result, "failed to allocate spare flush"); + } + + if (flush != NULL) { + struct flusher *flusher = pool_data; + + vdo_initialize_completion(&flush->completion, flusher->vdo, + VDO_FLUSH_COMPLETION); + } + + return flush; +} + +static void free_flush(void *element, void *pool_data __always_unused) +{ + vdo_free(element); +} + +/** + * vdo_make_flusher() - Make a flusher for a vdo. + * @vdo: The vdo which owns the flusher. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_make_flusher(struct vdo *vdo) +{ + int result = vdo_allocate(1, struct flusher, __func__, &vdo->flusher); + + if (result != VDO_SUCCESS) + return result; + + vdo->flusher->vdo = vdo; + vdo->flusher->thread_id = vdo->thread_config.packer_thread; + vdo_set_admin_state_code(&vdo->flusher->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + vdo_initialize_completion(&vdo->flusher->completion, vdo, + VDO_FLUSH_NOTIFICATION_COMPLETION); + + spin_lock_init(&vdo->flusher->lock); + bio_list_init(&vdo->flusher->waiting_flush_bios); + vdo->flusher->flush_pool = mempool_create(1, allocate_flush, free_flush, + vdo->flusher); + return ((vdo->flusher->flush_pool == NULL) ? -ENOMEM : VDO_SUCCESS); +} + +/** + * vdo_free_flusher() - Free a flusher. + * @flusher: The flusher to free. + */ +void vdo_free_flusher(struct flusher *flusher) +{ + if (flusher == NULL) + return; + + if (flusher->flush_pool != NULL) + mempool_destroy(vdo_forget(flusher->flush_pool)); + vdo_free(flusher); +} + +/** + * vdo_get_flusher_thread_id() - Get the ID of the thread on which flusher functions should be + * called. + * @flusher: The flusher to query. + * + * Return: The ID of the thread which handles the flusher. + */ +thread_id_t vdo_get_flusher_thread_id(struct flusher *flusher) +{ + return flusher->thread_id; +} + +static void notify_flush(struct flusher *flusher); +static void vdo_complete_flush(struct vdo_flush *flush); + +/** + * finish_notification() - Finish the notification process. + * @completion: The flusher completion. + * + * Finishes the notification process by checking if any flushes have completed and then starting + * the notification of the next flush request if one came in while the current notification was in + * progress. This callback is registered in flush_packer_callback(). + */ +static void finish_notification(struct vdo_completion *completion) +{ + struct flusher *flusher = as_flusher(completion); + + assert_on_flusher_thread(flusher, __func__); + + vdo_waitq_enqueue_waiter(&flusher->pending_flushes, + vdo_waitq_dequeue_waiter(&flusher->notifiers)); + vdo_complete_flushes(flusher); + if (vdo_waitq_has_waiters(&flusher->notifiers)) + notify_flush(flusher); +} + +/** + * flush_packer_callback() - Flush the packer. + * @completion: The flusher completion. + * + * Flushes the packer now that all of the logical and physical zones have been notified of the new + * flush request. This callback is registered in increment_generation(). + */ +static void flush_packer_callback(struct vdo_completion *completion) +{ + struct flusher *flusher = as_flusher(completion); + + vdo_increment_packer_flush_generation(flusher->vdo->packer); + vdo_launch_completion_callback(completion, finish_notification, + flusher->thread_id); +} + +/** + * increment_generation() - Increment the flush generation in a logical zone. + * @completion: The flusher as a completion. + * + * If there are more logical zones, go on to the next one, otherwise, prepare the physical zones. + * This callback is registered both in notify_flush() and in itself. + */ +static void increment_generation(struct vdo_completion *completion) +{ + struct flusher *flusher = as_flusher(completion); + struct logical_zone *zone = flusher->logical_zone_to_notify; + + vdo_increment_logical_zone_flush_generation(zone, flusher->notify_generation); + if (zone->next == NULL) { + vdo_launch_completion_callback(completion, flush_packer_callback, + flusher->thread_id); + return; + } + + flusher->logical_zone_to_notify = zone->next; + vdo_launch_completion_callback(completion, increment_generation, + flusher->logical_zone_to_notify->thread_id); +} + +/** + * notify_flush() - Launch a flush notification. + * @flusher: The flusher doing the notification. + */ +static void notify_flush(struct flusher *flusher) +{ + struct vdo_flush *flush = + vdo_waiter_as_flush(vdo_waitq_get_first_waiter(&flusher->notifiers)); + + flusher->notify_generation = flush->flush_generation; + flusher->logical_zone_to_notify = &flusher->vdo->logical_zones->zones[0]; + flusher->completion.requeue = true; + vdo_launch_completion_callback(&flusher->completion, increment_generation, + flusher->logical_zone_to_notify->thread_id); +} + +/** + * flush_vdo() - Start processing a flush request. + * @completion: A flush request (as a vdo_completion) + * + * This callback is registered in launch_flush(). + */ +static void flush_vdo(struct vdo_completion *completion) +{ + struct vdo_flush *flush = completion_as_vdo_flush(completion); + struct flusher *flusher = completion->vdo->flusher; + bool may_notify; + int result; + + assert_on_flusher_thread(flusher, __func__); + result = VDO_ASSERT(vdo_is_state_normal(&flusher->state), + "flusher is in normal operation"); + if (result != VDO_SUCCESS) { + vdo_enter_read_only_mode(flusher->vdo, result); + vdo_complete_flush(flush); + return; + } + + flush->flush_generation = flusher->flush_generation++; + may_notify = !vdo_waitq_has_waiters(&flusher->notifiers); + vdo_waitq_enqueue_waiter(&flusher->notifiers, &flush->waiter); + if (may_notify) + notify_flush(flusher); +} + +/** + * check_for_drain_complete() - Check whether the flusher has drained. + * @flusher: The flusher. + */ +static void check_for_drain_complete(struct flusher *flusher) +{ + bool drained; + + if (!vdo_is_state_draining(&flusher->state) || + vdo_waitq_has_waiters(&flusher->pending_flushes)) + return; + + spin_lock(&flusher->lock); + drained = bio_list_empty(&flusher->waiting_flush_bios); + spin_unlock(&flusher->lock); + + if (drained) + vdo_finish_draining(&flusher->state); +} + +/** + * vdo_complete_flushes() - Attempt to complete any flushes which might have finished. + * @flusher: The flusher. + */ +void vdo_complete_flushes(struct flusher *flusher) +{ + sequence_number_t oldest_active_generation = U64_MAX; + struct logical_zone *zone; + + assert_on_flusher_thread(flusher, __func__); + + for (zone = &flusher->vdo->logical_zones->zones[0]; zone != NULL; zone = zone->next) + oldest_active_generation = + min(oldest_active_generation, + READ_ONCE(zone->oldest_active_generation)); + + while (vdo_waitq_has_waiters(&flusher->pending_flushes)) { + struct vdo_flush *flush = + vdo_waiter_as_flush(vdo_waitq_get_first_waiter(&flusher->pending_flushes)); + + if (flush->flush_generation >= oldest_active_generation) + return; + + VDO_ASSERT_LOG_ONLY((flush->flush_generation == + flusher->first_unacknowledged_generation), + "acknowledged next expected flush, %llu, was: %llu", + (unsigned long long) flusher->first_unacknowledged_generation, + (unsigned long long) flush->flush_generation); + vdo_waitq_dequeue_waiter(&flusher->pending_flushes); + vdo_complete_flush(flush); + flusher->first_unacknowledged_generation++; + } + + check_for_drain_complete(flusher); +} + +/** + * vdo_dump_flusher() - Dump the flusher, in a thread-unsafe fashion. + * @flusher: The flusher. + */ +void vdo_dump_flusher(const struct flusher *flusher) +{ + vdo_log_info("struct flusher"); + vdo_log_info(" flush_generation=%llu first_unacknowledged_generation=%llu", + (unsigned long long) flusher->flush_generation, + (unsigned long long) flusher->first_unacknowledged_generation); + vdo_log_info(" notifiers queue is %s; pending_flushes queue is %s", + (vdo_waitq_has_waiters(&flusher->notifiers) ? "not empty" : "empty"), + (vdo_waitq_has_waiters(&flusher->pending_flushes) ? "not empty" : "empty")); +} + +/** + * initialize_flush() - Initialize a vdo_flush structure. + * @flush: The flush to initialize. + * @vdo: The vdo being flushed. + * + * Initializes a vdo_flush structure, transferring all the bios in the flusher's waiting_flush_bios + * list to it. The caller MUST already hold the lock. + */ +static void initialize_flush(struct vdo_flush *flush, struct vdo *vdo) +{ + bio_list_init(&flush->bios); + bio_list_merge(&flush->bios, &vdo->flusher->waiting_flush_bios); + bio_list_init(&vdo->flusher->waiting_flush_bios); +} + +static void launch_flush(struct vdo_flush *flush) +{ + struct vdo_completion *completion = &flush->completion; + + vdo_prepare_completion(completion, flush_vdo, flush_vdo, + completion->vdo->thread_config.packer_thread, NULL); + vdo_enqueue_completion(completion, VDO_DEFAULT_Q_FLUSH_PRIORITY); +} + +/** + * vdo_launch_flush() - Function called to start processing a flush request. + * @vdo: The vdo. + * @bio: The bio containing an empty flush request. + * + * This is called when we receive an empty flush bio from the block layer, and before acknowledging + * a non-empty bio with the FUA flag set. + */ +void vdo_launch_flush(struct vdo *vdo, struct bio *bio) +{ + /* + * Try to allocate a vdo_flush to represent the flush request. If the allocation fails, + * we'll deal with it later. + */ + struct vdo_flush *flush = mempool_alloc(vdo->flusher->flush_pool, GFP_NOWAIT); + struct flusher *flusher = vdo->flusher; + const struct admin_state_code *code = vdo_get_admin_state_code(&flusher->state); + + VDO_ASSERT_LOG_ONLY(!code->quiescent, "Flushing not allowed in state %s", + code->name); + + spin_lock(&flusher->lock); + + /* We have a new bio to start. Add it to the list. */ + bio_list_add(&flusher->waiting_flush_bios, bio); + + if (flush == NULL) { + spin_unlock(&flusher->lock); + return; + } + + /* We have flushes to start. Capture them in the vdo_flush structure. */ + initialize_flush(flush, vdo); + spin_unlock(&flusher->lock); + + /* Finish launching the flushes. */ + launch_flush(flush); +} + +/** + * release_flush() - Release a vdo_flush structure that has completed its work. + * @flush: The completed flush structure to re-use or free. + * + * If there are any pending flush requests whose vdo_flush allocation failed, they will be launched + * by immediately re-using the released vdo_flush. If there is no spare vdo_flush, the released + * structure will become the spare. Otherwise, the vdo_flush will be freed. + */ +static void release_flush(struct vdo_flush *flush) +{ + bool relaunch_flush; + struct flusher *flusher = flush->completion.vdo->flusher; + + spin_lock(&flusher->lock); + if (bio_list_empty(&flusher->waiting_flush_bios)) { + relaunch_flush = false; + } else { + /* We have flushes to start. Capture them in a flush request. */ + initialize_flush(flush, flusher->vdo); + relaunch_flush = true; + } + spin_unlock(&flusher->lock); + + if (relaunch_flush) { + /* Finish launching the flushes. */ + launch_flush(flush); + return; + } + + mempool_free(flush, flusher->flush_pool); +} + +/** + * vdo_complete_flush_callback() - Function called to complete and free a flush request, registered + * in vdo_complete_flush(). + * @completion: The flush request. + */ +static void vdo_complete_flush_callback(struct vdo_completion *completion) +{ + struct vdo_flush *flush = completion_as_vdo_flush(completion); + struct vdo *vdo = completion->vdo; + struct bio *bio; + + while ((bio = bio_list_pop(&flush->bios)) != NULL) { + /* + * We're not acknowledging this bio now, but we'll never touch it again, so this is + * the last chance to account for it. + */ + vdo_count_bios(&vdo->stats.bios_acknowledged, bio); + + /* Update the device, and send it on down... */ + bio_set_dev(bio, vdo_get_backing_device(vdo)); + atomic64_inc(&vdo->stats.flush_out); + submit_bio_noacct(bio); + } + + + /* + * Release the flush structure, freeing it, re-using it as the spare, or using it to launch + * any flushes that had to wait when allocations failed. + */ + release_flush(flush); +} + +/** + * select_bio_queue() - Select the bio queue on which to finish a flush request. + * @flusher: The flusher finishing the request. + */ +static thread_id_t select_bio_queue(struct flusher *flusher) +{ + struct vdo *vdo = flusher->vdo; + zone_count_t bio_threads = flusher->vdo->thread_config.bio_thread_count; + int interval; + + if (bio_threads == 1) + return vdo->thread_config.bio_threads[0]; + + interval = vdo->device_config->thread_counts.bio_rotation_interval; + if (flusher->flush_count == interval) { + flusher->flush_count = 1; + flusher->bio_queue_rotor = ((flusher->bio_queue_rotor + 1) % bio_threads); + } else { + flusher->flush_count++; + } + + return vdo->thread_config.bio_threads[flusher->bio_queue_rotor]; +} + +/** + * vdo_complete_flush() - Complete and free a vdo flush request. + * @flush: The flush request. + */ +static void vdo_complete_flush(struct vdo_flush *flush) +{ + struct vdo_completion *completion = &flush->completion; + + vdo_prepare_completion(completion, vdo_complete_flush_callback, + vdo_complete_flush_callback, + select_bio_queue(completion->vdo->flusher), NULL); + vdo_enqueue_completion(completion, BIO_Q_FLUSH_PRIORITY); +} + +/** + * initiate_drain() - Initiate a drain. + * + * Implements vdo_admin_initiator_fn. + */ +static void initiate_drain(struct admin_state *state) +{ + check_for_drain_complete(container_of(state, struct flusher, state)); +} + +/** + * vdo_drain_flusher() - Drain the flusher. + * @flusher: The flusher to drain. + * @completion: The completion to finish when the flusher has drained. + * + * Drains the flusher by preventing any more VIOs from entering the flusher and then flushing. The + * flusher will be left in the suspended state. + */ +void vdo_drain_flusher(struct flusher *flusher, struct vdo_completion *completion) +{ + assert_on_flusher_thread(flusher, __func__); + vdo_start_draining(&flusher->state, VDO_ADMIN_STATE_SUSPENDING, completion, + initiate_drain); +} + +/** + * vdo_resume_flusher() - Resume a flusher which has been suspended. + * @flusher: The flusher to resume. + * @parent: The completion to finish when the flusher has resumed. + */ +void vdo_resume_flusher(struct flusher *flusher, struct vdo_completion *parent) +{ + assert_on_flusher_thread(flusher, __func__); + vdo_continue_completion(parent, vdo_resume_if_quiescent(&flusher->state)); +} diff --git a/drivers/md/dm-vdo/flush.h b/drivers/md/dm-vdo/flush.h new file mode 100644 index 000000000000..97252d6656e0 --- /dev/null +++ b/drivers/md/dm-vdo/flush.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_FLUSH_H +#define VDO_FLUSH_H + +#include "funnel-workqueue.h" +#include "types.h" +#include "vio.h" +#include "wait-queue.h" + +/* A marker for tracking which journal entries are affected by a flush request. */ +struct vdo_flush { + /* The completion for enqueueing this flush request. */ + struct vdo_completion completion; + /* The flush bios covered by this request */ + struct bio_list bios; + /* The wait queue entry for this flush */ + struct vdo_waiter waiter; + /* Which flush this struct represents */ + sequence_number_t flush_generation; +}; + +struct flusher; + +int __must_check vdo_make_flusher(struct vdo *vdo); + +void vdo_free_flusher(struct flusher *flusher); + +thread_id_t __must_check vdo_get_flusher_thread_id(struct flusher *flusher); + +void vdo_complete_flushes(struct flusher *flusher); + +void vdo_dump_flusher(const struct flusher *flusher); + +void vdo_launch_flush(struct vdo *vdo, struct bio *bio); + +void vdo_drain_flusher(struct flusher *flusher, struct vdo_completion *completion); + +void vdo_resume_flusher(struct flusher *flusher, struct vdo_completion *parent); + +#endif /* VDO_FLUSH_H */ diff --git a/drivers/md/dm-vdo/funnel-queue.c b/drivers/md/dm-vdo/funnel-queue.c new file mode 100644 index 000000000000..a63b2f2bfd7d --- /dev/null +++ b/drivers/md/dm-vdo/funnel-queue.c @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "funnel-queue.h" + +#include "cpu.h" +#include "memory-alloc.h" +#include "permassert.h" + +int vdo_make_funnel_queue(struct funnel_queue **queue_ptr) +{ + int result; + struct funnel_queue *queue; + + result = vdo_allocate(1, struct funnel_queue, "funnel queue", &queue); + if (result != VDO_SUCCESS) + return result; + + /* + * Initialize the stub entry and put it in the queue, establishing the invariant that + * queue->newest and queue->oldest are never null. + */ + queue->stub.next = NULL; + queue->newest = &queue->stub; + queue->oldest = &queue->stub; + + *queue_ptr = queue; + return VDO_SUCCESS; +} + +void vdo_free_funnel_queue(struct funnel_queue *queue) +{ + vdo_free(queue); +} + +static struct funnel_queue_entry *get_oldest(struct funnel_queue *queue) +{ + /* + * Barrier requirements: We need a read barrier between reading a "next" field pointer + * value and reading anything it points to. There's an accompanying barrier in + * vdo_funnel_queue_put() between its caller setting up the entry and making it visible. + */ + struct funnel_queue_entry *oldest = queue->oldest; + struct funnel_queue_entry *next = READ_ONCE(oldest->next); + + if (oldest == &queue->stub) { + /* + * When the oldest entry is the stub and it has no successor, the queue is + * logically empty. + */ + if (next == NULL) + return NULL; + /* + * The stub entry has a successor, so the stub can be dequeued and ignored without + * breaking the queue invariants. + */ + oldest = next; + queue->oldest = oldest; + next = READ_ONCE(oldest->next); + } + + /* + * We have a non-stub candidate to dequeue. If it lacks a successor, we'll need to put the + * stub entry back on the queue first. + */ + if (next == NULL) { + struct funnel_queue_entry *newest = READ_ONCE(queue->newest); + + if (oldest != newest) { + /* + * Another thread has already swung queue->newest atomically, but not yet + * assigned previous->next. The queue is really still empty. + */ + return NULL; + } + + /* + * Put the stub entry back on the queue, ensuring a successor will eventually be + * seen. + */ + vdo_funnel_queue_put(queue, &queue->stub); + + /* Check again for a successor. */ + next = READ_ONCE(oldest->next); + if (next == NULL) { + /* + * We lost a race with a producer who swapped queue->newest before we did, + * but who hasn't yet updated previous->next. Try again later. + */ + return NULL; + } + } + + return oldest; +} + +/* + * Poll a queue, removing the oldest entry if the queue is not empty. This function must only be + * called from a single consumer thread. + */ +struct funnel_queue_entry *vdo_funnel_queue_poll(struct funnel_queue *queue) +{ + struct funnel_queue_entry *oldest = get_oldest(queue); + + if (oldest == NULL) + return oldest; + + /* + * Dequeue the oldest entry and return it. Only one consumer thread may call this function, + * so no locking, atomic operations, or fences are needed; queue->oldest is owned by the + * consumer and oldest->next is never used by a producer thread after it is swung from NULL + * to non-NULL. + */ + queue->oldest = READ_ONCE(oldest->next); + /* + * Make sure the caller sees the proper stored data for this entry. Since we've already + * fetched the entry pointer we stored in "queue->oldest", this also ensures that on entry + * to the next call we'll properly see the dependent data. + */ + smp_rmb(); + /* + * If "oldest" is a very light-weight work item, we'll be looking for the next one very + * soon, so prefetch it now. + */ + uds_prefetch_address(queue->oldest, true); + WRITE_ONCE(oldest->next, NULL); + return oldest; +} + +/* + * Check whether the funnel queue is empty or not. If the queue is in a transition state with one + * or more entries being added such that the list view is incomplete, this function will report the + * queue as empty. + */ +bool vdo_is_funnel_queue_empty(struct funnel_queue *queue) +{ + return get_oldest(queue) == NULL; +} + +/* + * Check whether the funnel queue is idle or not. If the queue has entries available to be + * retrieved, it is not idle. If the queue is in a transition state with one or more entries being + * added such that the list view is incomplete, it may not be possible to retrieve an entry with + * the vdo_funnel_queue_poll() function, but the queue will not be considered idle. + */ +bool vdo_is_funnel_queue_idle(struct funnel_queue *queue) +{ + /* + * Oldest is not the stub, so there's another entry, though if next is NULL we can't + * retrieve it yet. + */ + if (queue->oldest != &queue->stub) + return false; + + /* + * Oldest is the stub, but newest has been updated by _put(); either there's another, + * retrievable entry in the list, or the list is officially empty but in the intermediate + * state of having an entry added. + * + * Whether anything is retrievable depends on whether stub.next has been updated and become + * visible to us, but for idleness we don't care. And due to memory ordering in _put(), the + * update to newest would be visible to us at the same time or sooner. + */ + if (READ_ONCE(queue->newest) != &queue->stub) + return false; + + return true; +} diff --git a/drivers/md/dm-vdo/funnel-queue.h b/drivers/md/dm-vdo/funnel-queue.h new file mode 100644 index 000000000000..bde0f1deff98 --- /dev/null +++ b/drivers/md/dm-vdo/funnel-queue.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_FUNNEL_QUEUE_H +#define VDO_FUNNEL_QUEUE_H + +#include +#include + +/* + * A funnel queue is a simple (almost) lock-free queue that accepts entries from multiple threads + * (multi-producer) and delivers them to a single thread (single-consumer). "Funnel" is an attempt + * to evoke the image of requests from more than one producer being "funneled down" to a single + * consumer. + * + * This is an unsynchronized but thread-safe data structure when used as intended. There is no + * mechanism to ensure that only one thread is consuming from the queue. If more than one thread + * attempts to consume from the queue, the resulting behavior is undefined. Clients must not + * directly access or manipulate the internals of the queue, which are only exposed for the purpose + * of allowing the very simple enqueue operation to be inlined. + * + * The implementation requires that a funnel_queue_entry structure (a link pointer) is embedded in + * the queue entries, and pointers to those structures are used exclusively by the queue. No macros + * are defined to template the queue, so the offset of the funnel_queue_entry in the records placed + * in the queue must all be the same so the client can derive their structure pointer from the + * entry pointer returned by vdo_funnel_queue_poll(). + * + * Callers are wholly responsible for allocating and freeing the entries. Entries may be freed as + * soon as they are returned since this queue is not susceptible to the "ABA problem" present in + * many lock-free data structures. The queue is dynamically allocated to ensure cache-line + * alignment, but no other dynamic allocation is used. + * + * The algorithm is not actually 100% lock-free. There is a single point in vdo_funnel_queue_put() + * at which a preempted producer will prevent the consumers from seeing items added to the queue by + * later producers, and only if the queue is short enough or the consumer fast enough for it to + * reach what was the end of the queue at the time of the preemption. + * + * The consumer function, vdo_funnel_queue_poll(), will return NULL when the queue is empty. To + * wait for data to consume, spin (if safe) or combine the queue with a struct event_count to + * signal the presence of new entries. + */ + +/* This queue link structure must be embedded in client entries. */ +struct funnel_queue_entry { + /* The next (newer) entry in the queue. */ + struct funnel_queue_entry *next; +}; + +/* + * The dynamically allocated queue structure, which is allocated on a cache line boundary so the + * producer and consumer fields in the structure will land on separate cache lines. This should be + * consider opaque but it is exposed here so vdo_funnel_queue_put() can be inlined. + */ +struct __aligned(L1_CACHE_BYTES) funnel_queue { + /* + * The producers' end of the queue, an atomically exchanged pointer that will never be + * NULL. + */ + struct funnel_queue_entry *newest; + + /* The consumer's end of the queue, which is owned by the consumer and never NULL. */ + struct funnel_queue_entry *oldest __aligned(L1_CACHE_BYTES); + + /* A dummy entry used to provide the non-NULL invariants above. */ + struct funnel_queue_entry stub; +}; + +int __must_check vdo_make_funnel_queue(struct funnel_queue **queue_ptr); + +void vdo_free_funnel_queue(struct funnel_queue *queue); + +/* + * Put an entry on the end of the queue. + * + * The entry pointer must be to the struct funnel_queue_entry embedded in the caller's data + * structure. The caller must be able to derive the address of the start of their data structure + * from the pointer that passed in here, so every entry in the queue must have the struct + * funnel_queue_entry at the same offset within the client's structure. + */ +static inline void vdo_funnel_queue_put(struct funnel_queue *queue, + struct funnel_queue_entry *entry) +{ + struct funnel_queue_entry *previous; + + /* + * Barrier requirements: All stores relating to the entry ("next" pointer, containing data + * structure fields) must happen before the previous->next store making it visible to the + * consumer. Also, the entry's "next" field initialization to NULL must happen before any + * other producer threads can see the entry (the xchg) and try to update the "next" field. + * + * xchg implements a full barrier. + */ + WRITE_ONCE(entry->next, NULL); + previous = xchg(&queue->newest, entry); + /* + * Preemptions between these two statements hide the rest of the queue from the consumer, + * preventing consumption until the following assignment runs. + */ + WRITE_ONCE(previous->next, entry); +} + +struct funnel_queue_entry *__must_check vdo_funnel_queue_poll(struct funnel_queue *queue); + +bool __must_check vdo_is_funnel_queue_empty(struct funnel_queue *queue); + +bool __must_check vdo_is_funnel_queue_idle(struct funnel_queue *queue); + +#endif /* VDO_FUNNEL_QUEUE_H */ diff --git a/drivers/md/dm-vdo/funnel-workqueue.c b/drivers/md/dm-vdo/funnel-workqueue.c new file mode 100644 index 000000000000..ae11941c90a9 --- /dev/null +++ b/drivers/md/dm-vdo/funnel-workqueue.c @@ -0,0 +1,638 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "funnel-workqueue.h" + +#include +#include +#include +#include +#include +#include + +#include "funnel-queue.h" +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "permassert.h" +#include "string-utils.h" + +#include "completion.h" +#include "status-codes.h" + +static DEFINE_PER_CPU(unsigned int, service_queue_rotor); + +/** + * DOC: Work queue definition. + * + * There are two types of work queues: simple, with one worker thread, and round-robin, which uses + * a group of the former to do the work, and assigns work to them in round-robin fashion (roughly). + * Externally, both are represented via the same common sub-structure, though there's actually not + * a great deal of overlap between the two types internally. + */ +struct vdo_work_queue { + /* Name of just the work queue (e.g., "cpuQ12") */ + char *name; + bool round_robin_mode; + struct vdo_thread *owner; + /* Life cycle functions, etc */ + const struct vdo_work_queue_type *type; +}; + +struct simple_work_queue { + struct vdo_work_queue common; + struct funnel_queue *priority_lists[VDO_WORK_Q_MAX_PRIORITY + 1]; + void *private; + + /* + * The fields above are unchanged after setup but often read, and are good candidates for + * caching -- and if the max priority is 2, just fit in one x86-64 cache line if aligned. + * The fields below are often modified as we sleep and wake, so we want a separate cache + * line for performance. + */ + + /* Any (0 or 1) worker threads waiting for new work to do */ + wait_queue_head_t waiting_worker_threads ____cacheline_aligned; + /* Hack to reduce wakeup calls if the worker thread is running */ + atomic_t idle; + + /* These are infrequently used so in terms of performance we don't care where they land. */ + struct task_struct *thread; + /* Notify creator once worker has initialized */ + struct completion *started; +}; + +struct round_robin_work_queue { + struct vdo_work_queue common; + struct simple_work_queue **service_queues; + unsigned int num_service_queues; +}; + +static inline struct simple_work_queue *as_simple_work_queue(struct vdo_work_queue *queue) +{ + return ((queue == NULL) ? + NULL : container_of(queue, struct simple_work_queue, common)); +} + +static inline struct round_robin_work_queue *as_round_robin_work_queue(struct vdo_work_queue *queue) +{ + return ((queue == NULL) ? + NULL : + container_of(queue, struct round_robin_work_queue, common)); +} + +/* Processing normal completions. */ + +/* + * Dequeue and return the next waiting completion, if any. + * + * We scan the funnel queues from highest priority to lowest, once; there is therefore a race + * condition where a high-priority completion can be enqueued followed by a lower-priority one, and + * we'll grab the latter (but we'll catch the high-priority item on the next call). If strict + * enforcement of priorities becomes necessary, this function will need fixing. + */ +static struct vdo_completion *poll_for_completion(struct simple_work_queue *queue) +{ + int i; + + for (i = queue->common.type->max_priority; i >= 0; i--) { + struct funnel_queue_entry *link = vdo_funnel_queue_poll(queue->priority_lists[i]); + + if (link != NULL) + return container_of(link, struct vdo_completion, work_queue_entry_link); + } + + return NULL; +} + +static void enqueue_work_queue_completion(struct simple_work_queue *queue, + struct vdo_completion *completion) +{ + VDO_ASSERT_LOG_ONLY(completion->my_queue == NULL, + "completion %px (fn %px) to enqueue (%px) is not already queued (%px)", + completion, completion->callback, queue, completion->my_queue); + if (completion->priority == VDO_WORK_Q_DEFAULT_PRIORITY) + completion->priority = queue->common.type->default_priority; + + if (VDO_ASSERT(completion->priority <= queue->common.type->max_priority, + "priority is in range for queue") != VDO_SUCCESS) + completion->priority = 0; + + completion->my_queue = &queue->common; + + /* Funnel queue handles the synchronization for the put. */ + vdo_funnel_queue_put(queue->priority_lists[completion->priority], + &completion->work_queue_entry_link); + + /* + * Due to how funnel queue synchronization is handled (just atomic operations), the + * simplest safe implementation here would be to wake-up any waiting threads after + * enqueueing each item. Even if the funnel queue is not empty at the time of adding an + * item to the queue, the consumer thread may not see this since it is not guaranteed to + * have the same view of the queue as a producer thread. + * + * However, the above is wasteful so instead we attempt to minimize the number of thread + * wakeups. Using an idle flag, and careful ordering using memory barriers, we should be + * able to determine when the worker thread might be asleep or going to sleep. We use + * cmpxchg to try to take ownership (vs other producer threads) of the responsibility for + * waking the worker thread, so multiple wakeups aren't tried at once. + * + * This was tuned for some x86 boxes that were handy; it's untested whether doing the read + * first is any better or worse for other platforms, even other x86 configurations. + */ + smp_mb(); + if ((atomic_read(&queue->idle) != 1) || (atomic_cmpxchg(&queue->idle, 1, 0) != 1)) + return; + + /* There's a maximum of one thread in this list. */ + wake_up(&queue->waiting_worker_threads); +} + +static void run_start_hook(struct simple_work_queue *queue) +{ + if (queue->common.type->start != NULL) + queue->common.type->start(queue->private); +} + +static void run_finish_hook(struct simple_work_queue *queue) +{ + if (queue->common.type->finish != NULL) + queue->common.type->finish(queue->private); +} + +/* + * Wait for the next completion to process, or until kthread_should_stop indicates that it's time + * for us to shut down. + * + * If kthread_should_stop says it's time to stop but we have pending completions return a + * completion. + * + * Also update statistics relating to scheduler interactions. + */ +static struct vdo_completion *wait_for_next_completion(struct simple_work_queue *queue) +{ + struct vdo_completion *completion; + DEFINE_WAIT(wait); + + while (true) { + prepare_to_wait(&queue->waiting_worker_threads, &wait, + TASK_INTERRUPTIBLE); + /* + * Don't set the idle flag until a wakeup will not be lost. + * + * Force synchronization between setting the idle flag and checking the funnel + * queue; the producer side will do them in the reverse order. (There's still a + * race condition we've chosen to allow, because we've got a timeout below that + * unwedges us if we hit it, but this may narrow the window a little.) + */ + atomic_set(&queue->idle, 1); + smp_mb(); /* store-load barrier between "idle" and funnel queue */ + + completion = poll_for_completion(queue); + if (completion != NULL) + break; + + /* + * We need to check for thread-stop after setting TASK_INTERRUPTIBLE state up + * above. Otherwise, schedule() will put the thread to sleep and might miss a + * wakeup from kthread_stop() call in vdo_finish_work_queue(). + */ + if (kthread_should_stop()) + break; + + schedule(); + + /* + * Most of the time when we wake, it should be because there's work to do. If it + * was a spurious wakeup, continue looping. + */ + completion = poll_for_completion(queue); + if (completion != NULL) + break; + } + + finish_wait(&queue->waiting_worker_threads, &wait); + atomic_set(&queue->idle, 0); + + return completion; +} + +static void process_completion(struct simple_work_queue *queue, + struct vdo_completion *completion) +{ + if (VDO_ASSERT(completion->my_queue == &queue->common, + "completion %px from queue %px marked as being in this queue (%px)", + completion, queue, completion->my_queue) == VDO_SUCCESS) + completion->my_queue = NULL; + + vdo_run_completion(completion); +} + +static void service_work_queue(struct simple_work_queue *queue) +{ + run_start_hook(queue); + + while (true) { + struct vdo_completion *completion = poll_for_completion(queue); + + if (completion == NULL) + completion = wait_for_next_completion(queue); + + if (completion == NULL) { + /* No completions but kthread_should_stop() was triggered. */ + break; + } + + process_completion(queue, completion); + + /* + * Be friendly to a CPU that has other work to do, if the kernel has told us to. + * This speeds up some performance tests; that "other work" might include other VDO + * threads. + */ + if (need_resched()) + cond_resched(); + } + + run_finish_hook(queue); +} + +static int work_queue_runner(void *ptr) +{ + struct simple_work_queue *queue = ptr; + + complete(queue->started); + service_work_queue(queue); + return 0; +} + +/* Creation & teardown */ + +static void free_simple_work_queue(struct simple_work_queue *queue) +{ + unsigned int i; + + for (i = 0; i <= VDO_WORK_Q_MAX_PRIORITY; i++) + vdo_free_funnel_queue(queue->priority_lists[i]); + vdo_free(queue->common.name); + vdo_free(queue); +} + +static void free_round_robin_work_queue(struct round_robin_work_queue *queue) +{ + struct simple_work_queue **queue_table = queue->service_queues; + unsigned int count = queue->num_service_queues; + unsigned int i; + + queue->service_queues = NULL; + + for (i = 0; i < count; i++) + free_simple_work_queue(queue_table[i]); + vdo_free(queue_table); + vdo_free(queue->common.name); + vdo_free(queue); +} + +void vdo_free_work_queue(struct vdo_work_queue *queue) +{ + if (queue == NULL) + return; + + vdo_finish_work_queue(queue); + + if (queue->round_robin_mode) + free_round_robin_work_queue(as_round_robin_work_queue(queue)); + else + free_simple_work_queue(as_simple_work_queue(queue)); +} + +static int make_simple_work_queue(const char *thread_name_prefix, const char *name, + struct vdo_thread *owner, void *private, + const struct vdo_work_queue_type *type, + struct simple_work_queue **queue_ptr) +{ + DECLARE_COMPLETION_ONSTACK(started); + struct simple_work_queue *queue; + int i; + struct task_struct *thread = NULL; + int result; + + VDO_ASSERT_LOG_ONLY((type->max_priority <= VDO_WORK_Q_MAX_PRIORITY), + "queue priority count %u within limit %u", type->max_priority, + VDO_WORK_Q_MAX_PRIORITY); + + result = vdo_allocate(1, struct simple_work_queue, "simple work queue", &queue); + if (result != VDO_SUCCESS) + return result; + + queue->private = private; + queue->started = &started; + queue->common.type = type; + queue->common.owner = owner; + init_waitqueue_head(&queue->waiting_worker_threads); + + result = vdo_duplicate_string(name, "queue name", &queue->common.name); + if (result != VDO_SUCCESS) { + vdo_free(queue); + return -ENOMEM; + } + + for (i = 0; i <= type->max_priority; i++) { + result = vdo_make_funnel_queue(&queue->priority_lists[i]); + if (result != VDO_SUCCESS) { + free_simple_work_queue(queue); + return result; + } + } + + thread = kthread_run(work_queue_runner, queue, "%s:%s", thread_name_prefix, + queue->common.name); + if (IS_ERR(thread)) { + free_simple_work_queue(queue); + return (int) PTR_ERR(thread); + } + + queue->thread = thread; + + /* + * If we don't wait to ensure the thread is running VDO code, a quick kthread_stop (due to + * errors elsewhere) could cause it to never get as far as running VDO, skipping the + * cleanup code. + * + * Eventually we should just make that path safe too, and then we won't need this + * synchronization. + */ + wait_for_completion(&started); + + *queue_ptr = queue; + return VDO_SUCCESS; +} + +/** + * vdo_make_work_queue() - Create a work queue; if multiple threads are requested, completions will + * be distributed to them in round-robin fashion. + * + * Each queue is associated with a struct vdo_thread which has a single vdo thread id. Regardless + * of the actual number of queues and threads allocated here, code outside of the queue + * implementation will treat this as a single zone. + */ +int vdo_make_work_queue(const char *thread_name_prefix, const char *name, + struct vdo_thread *owner, const struct vdo_work_queue_type *type, + unsigned int thread_count, void *thread_privates[], + struct vdo_work_queue **queue_ptr) +{ + struct round_robin_work_queue *queue; + int result; + char thread_name[TASK_COMM_LEN]; + unsigned int i; + + if (thread_count == 1) { + struct simple_work_queue *simple_queue; + void *context = ((thread_privates != NULL) ? thread_privates[0] : NULL); + + result = make_simple_work_queue(thread_name_prefix, name, owner, context, + type, &simple_queue); + if (result == VDO_SUCCESS) + *queue_ptr = &simple_queue->common; + return result; + } + + result = vdo_allocate(1, struct round_robin_work_queue, "round-robin work queue", + &queue); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate(thread_count, struct simple_work_queue *, + "subordinate work queues", &queue->service_queues); + if (result != VDO_SUCCESS) { + vdo_free(queue); + return result; + } + + queue->num_service_queues = thread_count; + queue->common.round_robin_mode = true; + queue->common.owner = owner; + + result = vdo_duplicate_string(name, "queue name", &queue->common.name); + if (result != VDO_SUCCESS) { + vdo_free(queue->service_queues); + vdo_free(queue); + return -ENOMEM; + } + + *queue_ptr = &queue->common; + + for (i = 0; i < thread_count; i++) { + void *context = ((thread_privates != NULL) ? thread_privates[i] : NULL); + + snprintf(thread_name, sizeof(thread_name), "%s%u", name, i); + result = make_simple_work_queue(thread_name_prefix, thread_name, owner, + context, type, &queue->service_queues[i]); + if (result != VDO_SUCCESS) { + queue->num_service_queues = i; + /* Destroy previously created subordinates. */ + vdo_free_work_queue(vdo_forget(*queue_ptr)); + return result; + } + } + + return VDO_SUCCESS; +} + +static void finish_simple_work_queue(struct simple_work_queue *queue) +{ + if (queue->thread == NULL) + return; + + /* Tells the worker thread to shut down and waits for it to exit. */ + kthread_stop(queue->thread); + queue->thread = NULL; +} + +static void finish_round_robin_work_queue(struct round_robin_work_queue *queue) +{ + struct simple_work_queue **queue_table = queue->service_queues; + unsigned int count = queue->num_service_queues; + unsigned int i; + + for (i = 0; i < count; i++) + finish_simple_work_queue(queue_table[i]); +} + +/* No enqueueing of completions should be done once this function is called. */ +void vdo_finish_work_queue(struct vdo_work_queue *queue) +{ + if (queue == NULL) + return; + + if (queue->round_robin_mode) + finish_round_robin_work_queue(as_round_robin_work_queue(queue)); + else + finish_simple_work_queue(as_simple_work_queue(queue)); +} + +/* Debugging dumps */ + +static void dump_simple_work_queue(struct simple_work_queue *queue) +{ + const char *thread_status = "no threads"; + char task_state_report = '-'; + + if (queue->thread != NULL) { + task_state_report = task_state_to_char(queue->thread); + thread_status = atomic_read(&queue->idle) ? "idle" : "running"; + } + + vdo_log_info("workQ %px (%s) %s (%c)", &queue->common, queue->common.name, + thread_status, task_state_report); + + /* ->waiting_worker_threads wait queue status? anyone waiting? */ +} + +/* + * Write to the buffer some info about the completion, for logging. Since the common use case is + * dumping info about a lot of completions to syslog all at once, the format favors brevity over + * readability. + */ +void vdo_dump_work_queue(struct vdo_work_queue *queue) +{ + if (queue->round_robin_mode) { + struct round_robin_work_queue *round_robin = as_round_robin_work_queue(queue); + unsigned int i; + + for (i = 0; i < round_robin->num_service_queues; i++) + dump_simple_work_queue(round_robin->service_queues[i]); + } else { + dump_simple_work_queue(as_simple_work_queue(queue)); + } +} + +static void get_function_name(void *pointer, char *buffer, size_t buffer_length) +{ + if (pointer == NULL) { + /* + * Format "%ps" logs a null pointer as "(null)" with a bunch of leading spaces. We + * sometimes use this when logging lots of data; don't be so verbose. + */ + strscpy(buffer, "-", buffer_length); + } else { + /* + * Use a pragma to defeat gcc's format checking, which doesn't understand that + * "%ps" actually does support a precision spec in Linux kernel code. + */ + char *space; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat" + snprintf(buffer, buffer_length, "%.*ps", buffer_length - 1, pointer); +#pragma GCC diagnostic pop + + space = strchr(buffer, ' '); + if (space != NULL) + *space = '\0'; + } +} + +void vdo_dump_completion_to_buffer(struct vdo_completion *completion, char *buffer, + size_t length) +{ + size_t current_length = + scnprintf(buffer, length, "%.*s/", TASK_COMM_LEN, + (completion->my_queue == NULL ? "-" : completion->my_queue->name)); + + if (current_length < length - 1) { + get_function_name((void *) completion->callback, buffer + current_length, + length - current_length); + } +} + +/* Completion submission */ +/* + * If the completion has a timeout that has already passed, the timeout handler function may be + * invoked by this function. + */ +void vdo_enqueue_work_queue(struct vdo_work_queue *queue, + struct vdo_completion *completion) +{ + /* + * Convert the provided generic vdo_work_queue to the simple_work_queue to actually queue + * on. + */ + struct simple_work_queue *simple_queue = NULL; + + if (!queue->round_robin_mode) { + simple_queue = as_simple_work_queue(queue); + } else { + struct round_robin_work_queue *round_robin = as_round_robin_work_queue(queue); + + /* + * It shouldn't be a big deal if the same rotor gets used for multiple work queues. + * Any patterns that might develop are likely to be disrupted by random ordering of + * multiple completions and migration between cores, unless the load is so light as + * to be regular in ordering of tasks and the threads are confined to individual + * cores; with a load that light we won't care. + */ + unsigned int rotor = this_cpu_inc_return(service_queue_rotor); + unsigned int index = rotor % round_robin->num_service_queues; + + simple_queue = round_robin->service_queues[index]; + } + + enqueue_work_queue_completion(simple_queue, completion); +} + +/* Misc */ + +/* + * Return the work queue pointer recorded at initialization time in the work-queue stack handle + * initialized on the stack of the current thread, if any. + */ +static struct simple_work_queue *get_current_thread_work_queue(void) +{ + /* + * In interrupt context, if a vdo thread is what got interrupted, the calls below will find + * the queue for the thread which was interrupted. However, the interrupted thread may have + * been processing a completion, in which case starting to process another would violate + * our concurrency assumptions. + */ + if (in_interrupt()) + return NULL; + + if (kthread_func(current) != work_queue_runner) + /* Not a VDO work queue thread. */ + return NULL; + + return kthread_data(current); +} + +struct vdo_work_queue *vdo_get_current_work_queue(void) +{ + struct simple_work_queue *queue = get_current_thread_work_queue(); + + return (queue == NULL) ? NULL : &queue->common; +} + +struct vdo_thread *vdo_get_work_queue_owner(struct vdo_work_queue *queue) +{ + return queue->owner; +} + +/** + * vdo_get_work_queue_private_data() - Returns the private data for the current thread's work + * queue, or NULL if none or if the current thread is not a + * work queue thread. + */ +void *vdo_get_work_queue_private_data(void) +{ + struct simple_work_queue *queue = get_current_thread_work_queue(); + + return (queue != NULL) ? queue->private : NULL; +} + +bool vdo_work_queue_type_is(struct vdo_work_queue *queue, + const struct vdo_work_queue_type *type) +{ + return (queue->type == type); +} diff --git a/drivers/md/dm-vdo/funnel-workqueue.h b/drivers/md/dm-vdo/funnel-workqueue.h new file mode 100644 index 000000000000..b5be6e9e83bc --- /dev/null +++ b/drivers/md/dm-vdo/funnel-workqueue.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_WORK_QUEUE_H +#define VDO_WORK_QUEUE_H + +#include /* for TASK_COMM_LEN */ + +#include "types.h" + +enum { + MAX_VDO_WORK_QUEUE_NAME_LEN = TASK_COMM_LEN, +}; + +struct vdo_work_queue_type { + void (*start)(void *context); + void (*finish)(void *context); + enum vdo_completion_priority max_priority; + enum vdo_completion_priority default_priority; +}; + +struct vdo_completion; +struct vdo_thread; +struct vdo_work_queue; + +int vdo_make_work_queue(const char *thread_name_prefix, const char *name, + struct vdo_thread *owner, const struct vdo_work_queue_type *type, + unsigned int thread_count, void *thread_privates[], + struct vdo_work_queue **queue_ptr); + +void vdo_enqueue_work_queue(struct vdo_work_queue *queue, struct vdo_completion *completion); + +void vdo_finish_work_queue(struct vdo_work_queue *queue); + +void vdo_free_work_queue(struct vdo_work_queue *queue); + +void vdo_dump_work_queue(struct vdo_work_queue *queue); + +void vdo_dump_completion_to_buffer(struct vdo_completion *completion, char *buffer, + size_t length); + +void *vdo_get_work_queue_private_data(void); +struct vdo_work_queue *vdo_get_current_work_queue(void); +struct vdo_thread *vdo_get_work_queue_owner(struct vdo_work_queue *queue); + +bool __must_check vdo_work_queue_type_is(struct vdo_work_queue *queue, + const struct vdo_work_queue_type *type); + +#endif /* VDO_WORK_QUEUE_H */ diff --git a/drivers/md/dm-vdo/indexer/chapter-index.c b/drivers/md/dm-vdo/indexer/chapter-index.c new file mode 100644 index 000000000000..7e32a25d3f2f --- /dev/null +++ b/drivers/md/dm-vdo/indexer/chapter-index.c @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "chapter-index.h" + +#include "errors.h" +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "hash-utils.h" +#include "indexer.h" + +int uds_make_open_chapter_index(struct open_chapter_index **chapter_index, + const struct index_geometry *geometry, u64 volume_nonce) +{ + int result; + size_t memory_size; + struct open_chapter_index *index; + + result = vdo_allocate(1, struct open_chapter_index, "open chapter index", &index); + if (result != VDO_SUCCESS) + return result; + + /* + * The delta index will rebalance delta lists when memory gets tight, + * so give the chapter index one extra page. + */ + memory_size = ((geometry->index_pages_per_chapter + 1) * geometry->bytes_per_page); + index->geometry = geometry; + index->volume_nonce = volume_nonce; + result = uds_initialize_delta_index(&index->delta_index, 1, + geometry->delta_lists_per_chapter, + geometry->chapter_mean_delta, + geometry->chapter_payload_bits, + memory_size, 'm'); + if (result != UDS_SUCCESS) { + vdo_free(index); + return result; + } + + index->memory_size = index->delta_index.memory_size + sizeof(struct open_chapter_index); + *chapter_index = index; + return UDS_SUCCESS; +} + +void uds_free_open_chapter_index(struct open_chapter_index *chapter_index) +{ + if (chapter_index == NULL) + return; + + uds_uninitialize_delta_index(&chapter_index->delta_index); + vdo_free(chapter_index); +} + +/* Re-initialize an open chapter index for a new chapter. */ +void uds_empty_open_chapter_index(struct open_chapter_index *chapter_index, + u64 virtual_chapter_number) +{ + uds_reset_delta_index(&chapter_index->delta_index); + chapter_index->virtual_chapter_number = virtual_chapter_number; +} + +static inline bool was_entry_found(const struct delta_index_entry *entry, u32 address) +{ + return (!entry->at_end) && (entry->key == address); +} + +/* Associate a record name with the record page containing its metadata. */ +int uds_put_open_chapter_index_record(struct open_chapter_index *chapter_index, + const struct uds_record_name *name, + u32 page_number) +{ + int result; + struct delta_index_entry entry; + u32 address; + u32 list_number; + const u8 *found_name; + bool found; + const struct index_geometry *geometry = chapter_index->geometry; + u64 chapter_number = chapter_index->virtual_chapter_number; + u32 record_pages = geometry->record_pages_per_chapter; + + result = VDO_ASSERT(page_number < record_pages, + "Page number within chapter (%u) exceeds the maximum value %u", + page_number, record_pages); + if (result != VDO_SUCCESS) + return UDS_INVALID_ARGUMENT; + + address = uds_hash_to_chapter_delta_address(name, geometry); + list_number = uds_hash_to_chapter_delta_list(name, geometry); + result = uds_get_delta_index_entry(&chapter_index->delta_index, list_number, + address, name->name, &entry); + if (result != UDS_SUCCESS) + return result; + + found = was_entry_found(&entry, address); + result = VDO_ASSERT(!(found && entry.is_collision), + "Chunk appears more than once in chapter %llu", + (unsigned long long) chapter_number); + if (result != VDO_SUCCESS) + return UDS_BAD_STATE; + + found_name = (found ? name->name : NULL); + return uds_put_delta_index_entry(&entry, address, page_number, found_name); +} + +/* + * Pack a section of an open chapter index into a chapter index page. A range of delta lists + * (starting with a specified list index) is copied from the open chapter index into a memory page. + * The number of lists copied onto the page is returned to the caller on success. + * + * @chapter_index: The open chapter index + * @memory: The memory page to use + * @first_list: The first delta list number to be copied + * @last_page: If true, this is the last page of the chapter index and all the remaining lists must + * be packed onto this page + * @lists_packed: The number of delta lists that were packed onto this page + */ +int uds_pack_open_chapter_index_page(struct open_chapter_index *chapter_index, + u8 *memory, u32 first_list, bool last_page, + u32 *lists_packed) +{ + int result; + struct delta_index *delta_index = &chapter_index->delta_index; + struct delta_index_stats stats; + u64 nonce = chapter_index->volume_nonce; + u64 chapter_number = chapter_index->virtual_chapter_number; + const struct index_geometry *geometry = chapter_index->geometry; + u32 list_count = geometry->delta_lists_per_chapter; + unsigned int removals = 0; + struct delta_index_entry entry; + u32 next_list; + s32 list_number; + + for (;;) { + result = uds_pack_delta_index_page(delta_index, nonce, memory, + geometry->bytes_per_page, + chapter_number, first_list, + lists_packed); + if (result != UDS_SUCCESS) + return result; + + if ((first_list + *lists_packed) == list_count) { + /* All lists are packed. */ + break; + } else if (*lists_packed == 0) { + /* + * The next delta list does not fit on a page. This delta list will be + * removed. + */ + } else if (last_page) { + /* + * This is the last page and there are lists left unpacked, but all of the + * remaining lists must fit on the page. Find a list that contains entries + * and remove the entire list. Try the first list that does not fit. If it + * is empty, we will select the last list that already fits and has any + * entries. + */ + } else { + /* This page is done. */ + break; + } + + if (removals == 0) { + uds_get_delta_index_stats(delta_index, &stats); + vdo_log_warning("The chapter index for chapter %llu contains %llu entries with %llu collisions", + (unsigned long long) chapter_number, + (unsigned long long) stats.record_count, + (unsigned long long) stats.collision_count); + } + + list_number = *lists_packed; + do { + if (list_number < 0) + return UDS_OVERFLOW; + + next_list = first_list + list_number--, + result = uds_start_delta_index_search(delta_index, next_list, 0, + &entry); + if (result != UDS_SUCCESS) + return result; + + result = uds_next_delta_index_entry(&entry); + if (result != UDS_SUCCESS) + return result; + } while (entry.at_end); + + do { + result = uds_remove_delta_index_entry(&entry); + if (result != UDS_SUCCESS) + return result; + + removals++; + } while (!entry.at_end); + } + + if (removals > 0) { + vdo_log_warning("To avoid chapter index page overflow in chapter %llu, %u entries were removed from the chapter index", + (unsigned long long) chapter_number, removals); + } + + return UDS_SUCCESS; +} + +/* Make a new chapter index page, initializing it with the data from a given index_page buffer. */ +int uds_initialize_chapter_index_page(struct delta_index_page *index_page, + const struct index_geometry *geometry, + u8 *page_buffer, u64 volume_nonce) +{ + return uds_initialize_delta_index_page(index_page, volume_nonce, + geometry->chapter_mean_delta, + geometry->chapter_payload_bits, + page_buffer, geometry->bytes_per_page); +} + +/* Validate a chapter index page read during rebuild. */ +int uds_validate_chapter_index_page(const struct delta_index_page *index_page, + const struct index_geometry *geometry) +{ + int result; + const struct delta_index *delta_index = &index_page->delta_index; + u32 first = index_page->lowest_list_number; + u32 last = index_page->highest_list_number; + u32 list_number; + + /* We walk every delta list from start to finish. */ + for (list_number = first; list_number <= last; list_number++) { + struct delta_index_entry entry; + + result = uds_start_delta_index_search(delta_index, list_number - first, + 0, &entry); + if (result != UDS_SUCCESS) + return result; + + for (;;) { + result = uds_next_delta_index_entry(&entry); + if (result != UDS_SUCCESS) { + /* + * A random bit stream is highly likely to arrive here when we go + * past the end of the delta list. + */ + return result; + } + + if (entry.at_end) + break; + + /* Also make sure that the record page field contains a plausible value. */ + if (uds_get_delta_entry_value(&entry) >= + geometry->record_pages_per_chapter) { + /* + * Do not log this as an error. It happens in normal operation when + * we are doing a rebuild but haven't written the entire volume + * once. + */ + return UDS_CORRUPT_DATA; + } + } + } + return UDS_SUCCESS; +} + +/* + * Search a chapter index page for a record name, returning the record page number that may contain + * the name. + */ +int uds_search_chapter_index_page(struct delta_index_page *index_page, + const struct index_geometry *geometry, + const struct uds_record_name *name, + u16 *record_page_ptr) +{ + int result; + struct delta_index *delta_index = &index_page->delta_index; + u32 address = uds_hash_to_chapter_delta_address(name, geometry); + u32 delta_list_number = uds_hash_to_chapter_delta_list(name, geometry); + u32 sub_list_number = delta_list_number - index_page->lowest_list_number; + struct delta_index_entry entry; + + result = uds_get_delta_index_entry(delta_index, sub_list_number, address, + name->name, &entry); + if (result != UDS_SUCCESS) + return result; + + if (was_entry_found(&entry, address)) + *record_page_ptr = uds_get_delta_entry_value(&entry); + else + *record_page_ptr = NO_CHAPTER_INDEX_ENTRY; + + return UDS_SUCCESS; +} diff --git a/drivers/md/dm-vdo/indexer/chapter-index.h b/drivers/md/dm-vdo/indexer/chapter-index.h new file mode 100644 index 000000000000..be8bf2b675b1 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/chapter-index.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_CHAPTER_INDEX_H +#define UDS_CHAPTER_INDEX_H + +#include + +#include "delta-index.h" +#include "geometry.h" + +/* + * A chapter index for an open chapter is a mutable structure that tracks all the records that have + * been added to the chapter. A chapter index for a closed chapter is similar except that it is + * immutable because the contents of a closed chapter can never change, and the immutable structure + * is more efficient. Both types of chapter index are implemented with a delta index. + */ + +/* The value returned when no entry is found in the chapter index. */ +#define NO_CHAPTER_INDEX_ENTRY U16_MAX + +struct open_chapter_index { + const struct index_geometry *geometry; + struct delta_index delta_index; + u64 virtual_chapter_number; + u64 volume_nonce; + size_t memory_size; +}; + +int __must_check uds_make_open_chapter_index(struct open_chapter_index **chapter_index, + const struct index_geometry *geometry, + u64 volume_nonce); + +void uds_free_open_chapter_index(struct open_chapter_index *chapter_index); + +void uds_empty_open_chapter_index(struct open_chapter_index *chapter_index, + u64 virtual_chapter_number); + +int __must_check uds_put_open_chapter_index_record(struct open_chapter_index *chapter_index, + const struct uds_record_name *name, + u32 page_number); + +int __must_check uds_pack_open_chapter_index_page(struct open_chapter_index *chapter_index, + u8 *memory, u32 first_list, + bool last_page, u32 *lists_packed); + +int __must_check uds_initialize_chapter_index_page(struct delta_index_page *index_page, + const struct index_geometry *geometry, + u8 *page_buffer, u64 volume_nonce); + +int __must_check uds_validate_chapter_index_page(const struct delta_index_page *index_page, + const struct index_geometry *geometry); + +int __must_check uds_search_chapter_index_page(struct delta_index_page *index_page, + const struct index_geometry *geometry, + const struct uds_record_name *name, + u16 *record_page_ptr); + +#endif /* UDS_CHAPTER_INDEX_H */ diff --git a/drivers/md/dm-vdo/indexer/config.c b/drivers/md/dm-vdo/indexer/config.c new file mode 100644 index 000000000000..5532371b952f --- /dev/null +++ b/drivers/md/dm-vdo/indexer/config.c @@ -0,0 +1,376 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "config.h" + +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "string-utils.h" +#include "thread-utils.h" + +static const u8 INDEX_CONFIG_MAGIC[] = "ALBIC"; +static const u8 INDEX_CONFIG_VERSION_6_02[] = "06.02"; +static const u8 INDEX_CONFIG_VERSION_8_02[] = "08.02"; + +#define DEFAULT_VOLUME_READ_THREADS 2 +#define MAX_VOLUME_READ_THREADS 16 +#define INDEX_CONFIG_MAGIC_LENGTH (sizeof(INDEX_CONFIG_MAGIC) - 1) +#define INDEX_CONFIG_VERSION_LENGTH ((int)(sizeof(INDEX_CONFIG_VERSION_6_02) - 1)) + +static bool is_version(const u8 *version, u8 *buffer) +{ + return memcmp(version, buffer, INDEX_CONFIG_VERSION_LENGTH) == 0; +} + +static bool are_matching_configurations(struct uds_configuration *saved_config, + struct index_geometry *saved_geometry, + struct uds_configuration *user) +{ + struct index_geometry *geometry = user->geometry; + bool result = true; + + if (saved_geometry->record_pages_per_chapter != geometry->record_pages_per_chapter) { + vdo_log_error("Record pages per chapter (%u) does not match (%u)", + saved_geometry->record_pages_per_chapter, + geometry->record_pages_per_chapter); + result = false; + } + + if (saved_geometry->chapters_per_volume != geometry->chapters_per_volume) { + vdo_log_error("Chapter count (%u) does not match (%u)", + saved_geometry->chapters_per_volume, + geometry->chapters_per_volume); + result = false; + } + + if (saved_geometry->sparse_chapters_per_volume != geometry->sparse_chapters_per_volume) { + vdo_log_error("Sparse chapter count (%u) does not match (%u)", + saved_geometry->sparse_chapters_per_volume, + geometry->sparse_chapters_per_volume); + result = false; + } + + if (saved_config->cache_chapters != user->cache_chapters) { + vdo_log_error("Cache size (%u) does not match (%u)", + saved_config->cache_chapters, user->cache_chapters); + result = false; + } + + if (saved_config->volume_index_mean_delta != user->volume_index_mean_delta) { + vdo_log_error("Volume index mean delta (%u) does not match (%u)", + saved_config->volume_index_mean_delta, + user->volume_index_mean_delta); + result = false; + } + + if (saved_geometry->bytes_per_page != geometry->bytes_per_page) { + vdo_log_error("Bytes per page value (%zu) does not match (%zu)", + saved_geometry->bytes_per_page, geometry->bytes_per_page); + result = false; + } + + if (saved_config->sparse_sample_rate != user->sparse_sample_rate) { + vdo_log_error("Sparse sample rate (%u) does not match (%u)", + saved_config->sparse_sample_rate, + user->sparse_sample_rate); + result = false; + } + + if (saved_config->nonce != user->nonce) { + vdo_log_error("Nonce (%llu) does not match (%llu)", + (unsigned long long) saved_config->nonce, + (unsigned long long) user->nonce); + result = false; + } + + return result; +} + +/* Read the configuration and validate it against the provided one. */ +int uds_validate_config_contents(struct buffered_reader *reader, + struct uds_configuration *user_config) +{ + int result; + struct uds_configuration config; + struct index_geometry geometry; + u8 version_buffer[INDEX_CONFIG_VERSION_LENGTH]; + u32 bytes_per_page; + u8 buffer[sizeof(struct uds_configuration_6_02)]; + size_t offset = 0; + + result = uds_verify_buffered_data(reader, INDEX_CONFIG_MAGIC, + INDEX_CONFIG_MAGIC_LENGTH); + if (result != UDS_SUCCESS) + return result; + + result = uds_read_from_buffered_reader(reader, version_buffer, + INDEX_CONFIG_VERSION_LENGTH); + if (result != UDS_SUCCESS) + return vdo_log_error_strerror(result, "cannot read index config version"); + + if (!is_version(INDEX_CONFIG_VERSION_6_02, version_buffer) && + !is_version(INDEX_CONFIG_VERSION_8_02, version_buffer)) { + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "unsupported configuration version: '%.*s'", + INDEX_CONFIG_VERSION_LENGTH, + version_buffer); + } + + result = uds_read_from_buffered_reader(reader, buffer, sizeof(buffer)); + if (result != UDS_SUCCESS) + return vdo_log_error_strerror(result, "cannot read config data"); + + decode_u32_le(buffer, &offset, &geometry.record_pages_per_chapter); + decode_u32_le(buffer, &offset, &geometry.chapters_per_volume); + decode_u32_le(buffer, &offset, &geometry.sparse_chapters_per_volume); + decode_u32_le(buffer, &offset, &config.cache_chapters); + offset += sizeof(u32); + decode_u32_le(buffer, &offset, &config.volume_index_mean_delta); + decode_u32_le(buffer, &offset, &bytes_per_page); + geometry.bytes_per_page = bytes_per_page; + decode_u32_le(buffer, &offset, &config.sparse_sample_rate); + decode_u64_le(buffer, &offset, &config.nonce); + + result = VDO_ASSERT(offset == sizeof(struct uds_configuration_6_02), + "%zu bytes read but not decoded", + sizeof(struct uds_configuration_6_02) - offset); + if (result != VDO_SUCCESS) + return UDS_CORRUPT_DATA; + + if (is_version(INDEX_CONFIG_VERSION_6_02, version_buffer)) { + user_config->geometry->remapped_virtual = 0; + user_config->geometry->remapped_physical = 0; + } else { + u8 remapping[sizeof(u64) + sizeof(u64)]; + + result = uds_read_from_buffered_reader(reader, remapping, + sizeof(remapping)); + if (result != UDS_SUCCESS) + return vdo_log_error_strerror(result, "cannot read converted config"); + + offset = 0; + decode_u64_le(remapping, &offset, + &user_config->geometry->remapped_virtual); + decode_u64_le(remapping, &offset, + &user_config->geometry->remapped_physical); + } + + if (!are_matching_configurations(&config, &geometry, user_config)) { + vdo_log_warning("Supplied configuration does not match save"); + return UDS_NO_INDEX; + } + + return UDS_SUCCESS; +} + +/* + * Write the configuration to stable storage. If the superblock version is < 4, write the 6.02 + * version; otherwise write the 8.02 version, indicating the configuration is for an index that has + * been reduced by one chapter. + */ +int uds_write_config_contents(struct buffered_writer *writer, + struct uds_configuration *config, u32 version) +{ + int result; + struct index_geometry *geometry = config->geometry; + u8 buffer[sizeof(struct uds_configuration_8_02)]; + size_t offset = 0; + + result = uds_write_to_buffered_writer(writer, INDEX_CONFIG_MAGIC, + INDEX_CONFIG_MAGIC_LENGTH); + if (result != UDS_SUCCESS) + return result; + + /* + * If version is < 4, the index has not been reduced by a chapter so it must be written out + * as version 6.02 so that it is still compatible with older versions of UDS. + */ + if (version >= 4) { + result = uds_write_to_buffered_writer(writer, INDEX_CONFIG_VERSION_8_02, + INDEX_CONFIG_VERSION_LENGTH); + if (result != UDS_SUCCESS) + return result; + } else { + result = uds_write_to_buffered_writer(writer, INDEX_CONFIG_VERSION_6_02, + INDEX_CONFIG_VERSION_LENGTH); + if (result != UDS_SUCCESS) + return result; + } + + encode_u32_le(buffer, &offset, geometry->record_pages_per_chapter); + encode_u32_le(buffer, &offset, geometry->chapters_per_volume); + encode_u32_le(buffer, &offset, geometry->sparse_chapters_per_volume); + encode_u32_le(buffer, &offset, config->cache_chapters); + encode_u32_le(buffer, &offset, 0); + encode_u32_le(buffer, &offset, config->volume_index_mean_delta); + encode_u32_le(buffer, &offset, geometry->bytes_per_page); + encode_u32_le(buffer, &offset, config->sparse_sample_rate); + encode_u64_le(buffer, &offset, config->nonce); + + result = VDO_ASSERT(offset == sizeof(struct uds_configuration_6_02), + "%zu bytes encoded, of %zu expected", offset, + sizeof(struct uds_configuration_6_02)); + if (result != VDO_SUCCESS) + return result; + + if (version >= 4) { + encode_u64_le(buffer, &offset, geometry->remapped_virtual); + encode_u64_le(buffer, &offset, geometry->remapped_physical); + } + + return uds_write_to_buffered_writer(writer, buffer, offset); +} + +/* Compute configuration parameters that depend on memory size. */ +static int compute_memory_sizes(uds_memory_config_size_t mem_gb, bool sparse, + u32 *chapters_per_volume, u32 *record_pages_per_chapter, + u32 *sparse_chapters_per_volume) +{ + u32 reduced_chapters = 0; + u32 base_chapters; + + if (mem_gb == UDS_MEMORY_CONFIG_256MB) { + base_chapters = DEFAULT_CHAPTERS_PER_VOLUME; + *record_pages_per_chapter = SMALL_RECORD_PAGES_PER_CHAPTER; + } else if (mem_gb == UDS_MEMORY_CONFIG_512MB) { + base_chapters = DEFAULT_CHAPTERS_PER_VOLUME; + *record_pages_per_chapter = 2 * SMALL_RECORD_PAGES_PER_CHAPTER; + } else if (mem_gb == UDS_MEMORY_CONFIG_768MB) { + base_chapters = DEFAULT_CHAPTERS_PER_VOLUME; + *record_pages_per_chapter = 3 * SMALL_RECORD_PAGES_PER_CHAPTER; + } else if ((mem_gb >= 1) && (mem_gb <= UDS_MEMORY_CONFIG_MAX)) { + base_chapters = mem_gb * DEFAULT_CHAPTERS_PER_VOLUME; + *record_pages_per_chapter = DEFAULT_RECORD_PAGES_PER_CHAPTER; + } else if (mem_gb == UDS_MEMORY_CONFIG_REDUCED_256MB) { + reduced_chapters = 1; + base_chapters = DEFAULT_CHAPTERS_PER_VOLUME; + *record_pages_per_chapter = SMALL_RECORD_PAGES_PER_CHAPTER; + } else if (mem_gb == UDS_MEMORY_CONFIG_REDUCED_512MB) { + reduced_chapters = 1; + base_chapters = DEFAULT_CHAPTERS_PER_VOLUME; + *record_pages_per_chapter = 2 * SMALL_RECORD_PAGES_PER_CHAPTER; + } else if (mem_gb == UDS_MEMORY_CONFIG_REDUCED_768MB) { + reduced_chapters = 1; + base_chapters = DEFAULT_CHAPTERS_PER_VOLUME; + *record_pages_per_chapter = 3 * SMALL_RECORD_PAGES_PER_CHAPTER; + } else if ((mem_gb >= 1 + UDS_MEMORY_CONFIG_REDUCED) && + (mem_gb <= UDS_MEMORY_CONFIG_REDUCED_MAX)) { + reduced_chapters = 1; + base_chapters = ((mem_gb - UDS_MEMORY_CONFIG_REDUCED) * + DEFAULT_CHAPTERS_PER_VOLUME); + *record_pages_per_chapter = DEFAULT_RECORD_PAGES_PER_CHAPTER; + } else { + vdo_log_error("received invalid memory size"); + return -EINVAL; + } + + if (sparse) { + /* Make 95% of chapters sparse, allowing 10x more records. */ + *sparse_chapters_per_volume = (19 * base_chapters) / 2; + base_chapters *= 10; + } else { + *sparse_chapters_per_volume = 0; + } + + *chapters_per_volume = base_chapters - reduced_chapters; + return UDS_SUCCESS; +} + +static unsigned int __must_check normalize_zone_count(unsigned int requested) +{ + unsigned int zone_count = requested; + + if (zone_count == 0) + zone_count = num_online_cpus() / 2; + + if (zone_count < 1) + zone_count = 1; + + if (zone_count > MAX_ZONES) + zone_count = MAX_ZONES; + + vdo_log_info("Using %u indexing zone%s for concurrency.", + zone_count, zone_count == 1 ? "" : "s"); + return zone_count; +} + +static unsigned int __must_check normalize_read_threads(unsigned int requested) +{ + unsigned int read_threads = requested; + + if (read_threads < 1) + read_threads = DEFAULT_VOLUME_READ_THREADS; + + if (read_threads > MAX_VOLUME_READ_THREADS) + read_threads = MAX_VOLUME_READ_THREADS; + + return read_threads; +} + +int uds_make_configuration(const struct uds_parameters *params, + struct uds_configuration **config_ptr) +{ + struct uds_configuration *config; + u32 chapters_per_volume = 0; + u32 record_pages_per_chapter = 0; + u32 sparse_chapters_per_volume = 0; + int result; + + result = compute_memory_sizes(params->memory_size, params->sparse, + &chapters_per_volume, &record_pages_per_chapter, + &sparse_chapters_per_volume); + if (result != UDS_SUCCESS) + return result; + + result = vdo_allocate(1, struct uds_configuration, __func__, &config); + if (result != VDO_SUCCESS) + return result; + + result = uds_make_index_geometry(DEFAULT_BYTES_PER_PAGE, record_pages_per_chapter, + chapters_per_volume, sparse_chapters_per_volume, + 0, 0, &config->geometry); + if (result != UDS_SUCCESS) { + uds_free_configuration(config); + return result; + } + + config->zone_count = normalize_zone_count(params->zone_count); + config->read_threads = normalize_read_threads(params->read_threads); + + config->cache_chapters = DEFAULT_CACHE_CHAPTERS; + config->volume_index_mean_delta = DEFAULT_VOLUME_INDEX_MEAN_DELTA; + config->sparse_sample_rate = (params->sparse ? DEFAULT_SPARSE_SAMPLE_RATE : 0); + config->nonce = params->nonce; + config->bdev = params->bdev; + config->offset = params->offset; + config->size = params->size; + + *config_ptr = config; + return UDS_SUCCESS; +} + +void uds_free_configuration(struct uds_configuration *config) +{ + if (config != NULL) { + uds_free_index_geometry(config->geometry); + vdo_free(config); + } +} + +void uds_log_configuration(struct uds_configuration *config) +{ + struct index_geometry *geometry = config->geometry; + + vdo_log_debug("Configuration:"); + vdo_log_debug(" Record pages per chapter: %10u", geometry->record_pages_per_chapter); + vdo_log_debug(" Chapters per volume: %10u", geometry->chapters_per_volume); + vdo_log_debug(" Sparse chapters per volume: %10u", geometry->sparse_chapters_per_volume); + vdo_log_debug(" Cache size (chapters): %10u", config->cache_chapters); + vdo_log_debug(" Volume index mean delta: %10u", config->volume_index_mean_delta); + vdo_log_debug(" Bytes per page: %10zu", geometry->bytes_per_page); + vdo_log_debug(" Sparse sample rate: %10u", config->sparse_sample_rate); + vdo_log_debug(" Nonce: %llu", (unsigned long long) config->nonce); +} diff --git a/drivers/md/dm-vdo/indexer/config.h b/drivers/md/dm-vdo/indexer/config.h new file mode 100644 index 000000000000..08507dc2f7a1 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/config.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_CONFIG_H +#define UDS_CONFIG_H + +#include "geometry.h" +#include "indexer.h" +#include "io-factory.h" + +/* + * The uds_configuration records a variety of parameters used to configure a new UDS index. Some + * parameters are provided by the client, while others are fixed or derived from user-supplied + * values. It is created when an index is created, and it is recorded in the index metadata. + */ + +enum { + DEFAULT_VOLUME_INDEX_MEAN_DELTA = 4096, + DEFAULT_CACHE_CHAPTERS = 7, + DEFAULT_SPARSE_SAMPLE_RATE = 32, + MAX_ZONES = 16, +}; + +/* A set of configuration parameters for the indexer. */ +struct uds_configuration { + /* Storage device for the index */ + struct block_device *bdev; + + /* The maximum allowable size of the index */ + size_t size; + + /* The offset where the index should start */ + off_t offset; + + /* Parameters for the volume */ + + /* The volume layout */ + struct index_geometry *geometry; + + /* Index owner's nonce */ + u64 nonce; + + /* The number of threads used to process index requests */ + unsigned int zone_count; + + /* The number of threads used to read volume pages */ + unsigned int read_threads; + + /* Size of the page cache and sparse chapter index cache in chapters */ + u32 cache_chapters; + + /* Parameters for the volume index */ + + /* The mean delta for the volume index */ + u32 volume_index_mean_delta; + + /* Sampling rate for sparse indexing */ + u32 sparse_sample_rate; +}; + +/* On-disk structure of data for a version 8.02 index. */ +struct uds_configuration_8_02 { + /* Smaller (16), Small (64) or large (256) indices */ + u32 record_pages_per_chapter; + /* Total number of chapters per volume */ + u32 chapters_per_volume; + /* Number of sparse chapters per volume */ + u32 sparse_chapters_per_volume; + /* Size of the page cache, in chapters */ + u32 cache_chapters; + /* Unused field */ + u32 unused; + /* The volume index mean delta to use */ + u32 volume_index_mean_delta; + /* Size of a page, used for both record pages and index pages */ + u32 bytes_per_page; + /* Sampling rate for sparse indexing */ + u32 sparse_sample_rate; + /* Index owner's nonce */ + u64 nonce; + /* Virtual chapter remapped from physical chapter 0 */ + u64 remapped_virtual; + /* New physical chapter which remapped chapter was moved to */ + u64 remapped_physical; +} __packed; + +/* On-disk structure of data for a version 6.02 index. */ +struct uds_configuration_6_02 { + /* Smaller (16), Small (64) or large (256) indices */ + u32 record_pages_per_chapter; + /* Total number of chapters per volume */ + u32 chapters_per_volume; + /* Number of sparse chapters per volume */ + u32 sparse_chapters_per_volume; + /* Size of the page cache, in chapters */ + u32 cache_chapters; + /* Unused field */ + u32 unused; + /* The volume index mean delta to use */ + u32 volume_index_mean_delta; + /* Size of a page, used for both record pages and index pages */ + u32 bytes_per_page; + /* Sampling rate for sparse indexing */ + u32 sparse_sample_rate; + /* Index owner's nonce */ + u64 nonce; +} __packed; + +int __must_check uds_make_configuration(const struct uds_parameters *params, + struct uds_configuration **config_ptr); + +void uds_free_configuration(struct uds_configuration *config); + +int __must_check uds_validate_config_contents(struct buffered_reader *reader, + struct uds_configuration *config); + +int __must_check uds_write_config_contents(struct buffered_writer *writer, + struct uds_configuration *config, u32 version); + +void uds_log_configuration(struct uds_configuration *config); + +#endif /* UDS_CONFIG_H */ diff --git a/drivers/md/dm-vdo/indexer/delta-index.c b/drivers/md/dm-vdo/indexer/delta-index.c new file mode 100644 index 000000000000..0ac2443f0df3 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/delta-index.c @@ -0,0 +1,1970 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ +#include "delta-index.h" + +#include +#include +#include +#include +#include + +#include "cpu.h" +#include "errors.h" +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "permassert.h" +#include "string-utils.h" +#include "time-utils.h" + +#include "config.h" +#include "indexer.h" + +/* + * The entries in a delta index could be stored in a single delta list, but to reduce search times + * and update costs it uses multiple delta lists. These lists are stored in a single chunk of + * memory managed by the delta_zone structure. The delta_zone can move the data around within its + * memory, so the location of each delta list is recorded as a bit offset into the memory. Because + * the volume index can contain over a million delta lists, we want to be efficient with the size + * of the delta list header information. This information is encoded into 16 bytes per list. The + * volume index delta list memory can easily exceed 4 gigabits, so a 64 bit value is needed to + * address the memory. The volume index delta lists average around 6 kilobits, so 16 bits are + * sufficient to store the size of a delta list. + * + * Each delta list is stored as a bit stream. Within the delta list encoding, bits and bytes are + * numbered in little endian order. Within a byte, bit 0 is the least significant bit (0x1), and + * bit 7 is the most significant bit (0x80). Within a bit stream, bit 7 is the most significant bit + * of byte 0, and bit 8 is the least significant bit of byte 1. Within a byte array, a byte's + * number corresponds to its index in the array. + * + * A standard delta list entry is stored as a fixed length payload (the value) followed by a + * variable length key (the delta). A collision entry is used when two block names have the same + * delta list address. A collision entry always follows a standard entry for the hash with which it + * collides, and is encoded with DELTA == 0 with an additional 256 bits field at the end, + * containing the full block name. An entry with a delta of 0 at the beginning of a delta list + * indicates a normal entry. + * + * The delta in each entry is encoded with a variable-length Huffman code to minimize the memory + * used by small deltas. The Huffman code is specified by three parameters, which can be computed + * from the desired mean delta when the index is full. (See compute_coding_constants() for + * details.) + * + * The bit field utilities used to read and write delta entries assume that it is possible to read + * some bytes beyond the end of the bit field, so a delta_zone memory allocation is guarded by two + * invalid delta lists to prevent reading outside the delta_zone memory. The valid delta lists are + * numbered 1 to N, and the guard lists are numbered 0 and N+1. The function to decode the bit + * stream include a step that skips over bits set to 0 until the first 1 bit is found. A corrupted + * delta list could cause this step to run off the end of the delta_zone memory, so as extra + * protection against this happening, the tail guard list is set to all ones. + * + * The delta_index supports two different forms. The mutable form is created by + * uds_initialize_delta_index(), and is used for the volume index and for open chapter indexes. The + * immutable form is created by uds_initialize_delta_index_page(), and is used for closed (and + * cached) chapter index pages. The immutable form does not allocate delta list headers or + * temporary offsets, and thus is somewhat more memory efficient. + */ + +/* + * This is the largest field size supported by get_field() and set_field(). Any field that is + * larger is not guaranteed to fit in a single byte-aligned u32. + */ +#define MAX_FIELD_BITS ((sizeof(u32) - 1) * BITS_PER_BYTE + 1) + +/* + * This is the largest field size supported by get_big_field() and set_big_field(). Any field that + * is larger is not guaranteed to fit in a single byte-aligned u64. + */ +#define MAX_BIG_FIELD_BITS ((sizeof(u64) - 1) * BITS_PER_BYTE + 1) + +/* + * This is the number of guard bytes needed at the end of the memory byte array when using the bit + * utilities. These utilities call get_big_field() and set_big_field(), which can access up to 7 + * bytes beyond the end of the desired field. The definition is written to make it clear how this + * value is derived. + */ +#define POST_FIELD_GUARD_BYTES (sizeof(u64) - 1) + +/* The number of guard bits that are needed in the tail guard list */ +#define GUARD_BITS (POST_FIELD_GUARD_BYTES * BITS_PER_BYTE) + +/* + * The maximum size of a single delta list in bytes. We count guard bytes in this value because a + * buffer of this size can be used with move_bits(). + */ +#define DELTA_LIST_MAX_BYTE_COUNT \ + ((U16_MAX + BITS_PER_BYTE) / BITS_PER_BYTE + POST_FIELD_GUARD_BYTES) + +/* The number of extra bytes and bits needed to store a collision entry */ +#define COLLISION_BYTES UDS_RECORD_NAME_SIZE +#define COLLISION_BITS (COLLISION_BYTES * BITS_PER_BYTE) + +/* + * Immutable delta lists are packed into pages containing a header that encodes the delta list + * information into 19 bits per list (64KB bit offset). + */ +#define IMMUTABLE_HEADER_SIZE 19 + +/* + * Constants and structures for the saved delta index. "DI" is for delta_index, and -##### is a + * number to increment when the format of the data changes. + */ +#define MAGIC_SIZE 8 + +static const char DELTA_INDEX_MAGIC[] = "DI-00002"; + +struct delta_index_header { + char magic[MAGIC_SIZE]; + u32 zone_number; + u32 zone_count; + u32 first_list; + u32 list_count; + u64 record_count; + u64 collision_count; +}; + +/* + * Header data used for immutable delta index pages. This data is followed by the delta list offset + * table. + */ +struct delta_page_header { + /* Externally-defined nonce */ + u64 nonce; + /* The virtual chapter number */ + u64 virtual_chapter_number; + /* Index of the first delta list on the page */ + u16 first_list; + /* Number of delta lists on the page */ + u16 list_count; +} __packed; + +static inline u64 get_delta_list_byte_start(const struct delta_list *delta_list) +{ + return delta_list->start / BITS_PER_BYTE; +} + +static inline u16 get_delta_list_byte_size(const struct delta_list *delta_list) +{ + unsigned int bit_offset = delta_list->start % BITS_PER_BYTE; + + return BITS_TO_BYTES(bit_offset + delta_list->size); +} + +static void rebalance_delta_zone(const struct delta_zone *delta_zone, u32 first, + u32 last) +{ + struct delta_list *delta_list; + u64 new_start; + + if (first == last) { + /* Only one list is moving, and we know there is space. */ + delta_list = &delta_zone->delta_lists[first]; + new_start = delta_zone->new_offsets[first]; + if (delta_list->start != new_start) { + u64 source; + u64 destination; + + source = get_delta_list_byte_start(delta_list); + delta_list->start = new_start; + destination = get_delta_list_byte_start(delta_list); + memmove(delta_zone->memory + destination, + delta_zone->memory + source, + get_delta_list_byte_size(delta_list)); + } + } else { + /* + * There is more than one list. Divide the problem in half, and use recursive calls + * to process each half. Note that after this computation, first <= middle, and + * middle < last. + */ + u32 middle = (first + last) / 2; + + delta_list = &delta_zone->delta_lists[middle]; + new_start = delta_zone->new_offsets[middle]; + + /* + * The direction that our middle list is moving determines which half of the + * problem must be processed first. + */ + if (new_start > delta_list->start) { + rebalance_delta_zone(delta_zone, middle + 1, last); + rebalance_delta_zone(delta_zone, first, middle); + } else { + rebalance_delta_zone(delta_zone, first, middle); + rebalance_delta_zone(delta_zone, middle + 1, last); + } + } +} + +static inline size_t get_zone_memory_size(unsigned int zone_count, size_t memory_size) +{ + /* Round up so that each zone is a multiple of 64K in size. */ + size_t ALLOC_BOUNDARY = 64 * 1024; + + return (memory_size / zone_count + ALLOC_BOUNDARY - 1) & -ALLOC_BOUNDARY; +} + +void uds_reset_delta_index(const struct delta_index *delta_index) +{ + unsigned int z; + + /* + * Initialize all delta lists to be empty. We keep 2 extra delta list descriptors, one + * before the first real entry and one after so that we don't need to bounds check the + * array access when calculating preceding and following gap sizes. + */ + for (z = 0; z < delta_index->zone_count; z++) { + u64 list_bits; + u64 spacing; + u64 offset; + unsigned int i; + struct delta_zone *zone = &delta_index->delta_zones[z]; + struct delta_list *delta_lists = zone->delta_lists; + + /* Zeroing the delta list headers initializes the head guard list correctly. */ + memset(delta_lists, 0, + (zone->list_count + 2) * sizeof(struct delta_list)); + + /* Set all the bits in the end guard list. */ + list_bits = (u64) zone->size * BITS_PER_BYTE - GUARD_BITS; + delta_lists[zone->list_count + 1].start = list_bits; + delta_lists[zone->list_count + 1].size = GUARD_BITS; + memset(zone->memory + (list_bits / BITS_PER_BYTE), ~0, + POST_FIELD_GUARD_BYTES); + + /* Evenly space out the real delta lists by setting regular offsets. */ + spacing = list_bits / zone->list_count; + offset = spacing / 2; + for (i = 1; i <= zone->list_count; i++) { + delta_lists[i].start = offset; + offset += spacing; + } + + /* Update the statistics. */ + zone->discard_count += zone->record_count; + zone->record_count = 0; + zone->collision_count = 0; + } +} + +/* Compute the Huffman coding parameters for the given mean delta. The Huffman code is specified by + * three parameters: + * + * MINBITS The number of bits in the smallest code + * BASE The number of values coded using a code of length MINBITS + * INCR The number of values coded by using one additional bit + * + * These parameters are related by this equation: + * + * BASE + INCR == 1 << MINBITS + * + * The math for the Huffman code of an exponential distribution says that + * + * INCR = log(2) * MEAN_DELTA + * + * Then use the smallest MINBITS value so that + * + * (1 << MINBITS) > INCR + * + * And then + * + * BASE = (1 << MINBITS) - INCR + * + * Now the index can generate a code such that + * - The first BASE values code using MINBITS bits. + * - The next INCR values code using MINBITS+1 bits. + * - The next INCR values code using MINBITS+2 bits. + * - (and so on). + */ +static void compute_coding_constants(u32 mean_delta, u16 *min_bits, u32 *min_keys, u32 *incr_keys) +{ + /* + * We want to compute the rounded value of log(2) * mean_delta. Since we cannot always use + * floating point, use a really good integer approximation. + */ + *incr_keys = (836158UL * mean_delta + 603160UL) / 1206321UL; + *min_bits = bits_per(*incr_keys + 1); + *min_keys = (1 << *min_bits) - *incr_keys; +} + +void uds_uninitialize_delta_index(struct delta_index *delta_index) +{ + unsigned int z; + + if (delta_index->delta_zones == NULL) + return; + + for (z = 0; z < delta_index->zone_count; z++) { + vdo_free(vdo_forget(delta_index->delta_zones[z].new_offsets)); + vdo_free(vdo_forget(delta_index->delta_zones[z].delta_lists)); + vdo_free(vdo_forget(delta_index->delta_zones[z].memory)); + } + + vdo_free(delta_index->delta_zones); + memset(delta_index, 0, sizeof(struct delta_index)); +} + +static int initialize_delta_zone(struct delta_zone *delta_zone, size_t size, + u32 first_list, u32 list_count, u32 mean_delta, + u32 payload_bits, u8 tag) +{ + int result; + + result = vdo_allocate(size, u8, "delta list", &delta_zone->memory); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate(list_count + 2, u64, "delta list temp", + &delta_zone->new_offsets); + if (result != VDO_SUCCESS) + return result; + + /* Allocate the delta lists. */ + result = vdo_allocate(list_count + 2, struct delta_list, "delta lists", + &delta_zone->delta_lists); + if (result != VDO_SUCCESS) + return result; + + compute_coding_constants(mean_delta, &delta_zone->min_bits, + &delta_zone->min_keys, &delta_zone->incr_keys); + delta_zone->value_bits = payload_bits; + delta_zone->buffered_writer = NULL; + delta_zone->size = size; + delta_zone->rebalance_time = 0; + delta_zone->rebalance_count = 0; + delta_zone->record_count = 0; + delta_zone->collision_count = 0; + delta_zone->discard_count = 0; + delta_zone->overflow_count = 0; + delta_zone->first_list = first_list; + delta_zone->list_count = list_count; + delta_zone->tag = tag; + + return UDS_SUCCESS; +} + +int uds_initialize_delta_index(struct delta_index *delta_index, unsigned int zone_count, + u32 list_count, u32 mean_delta, u32 payload_bits, + size_t memory_size, u8 tag) +{ + int result; + unsigned int z; + size_t zone_memory; + + result = vdo_allocate(zone_count, struct delta_zone, "Delta Index Zones", + &delta_index->delta_zones); + if (result != VDO_SUCCESS) + return result; + + delta_index->zone_count = zone_count; + delta_index->list_count = list_count; + delta_index->lists_per_zone = DIV_ROUND_UP(list_count, zone_count); + delta_index->memory_size = 0; + delta_index->mutable = true; + delta_index->tag = tag; + + for (z = 0; z < zone_count; z++) { + u32 lists_in_zone = delta_index->lists_per_zone; + u32 first_list_in_zone = z * lists_in_zone; + + if (z == zone_count - 1) { + /* + * The last zone gets fewer lists if zone_count doesn't evenly divide + * list_count. We'll have an underflow if the assertion below doesn't hold. + */ + if (delta_index->list_count <= first_list_in_zone) { + uds_uninitialize_delta_index(delta_index); + return vdo_log_error_strerror(UDS_INVALID_ARGUMENT, + "%u delta lists not enough for %u zones", + list_count, zone_count); + } + lists_in_zone = delta_index->list_count - first_list_in_zone; + } + + zone_memory = get_zone_memory_size(zone_count, memory_size); + result = initialize_delta_zone(&delta_index->delta_zones[z], zone_memory, + first_list_in_zone, lists_in_zone, + mean_delta, payload_bits, tag); + if (result != UDS_SUCCESS) { + uds_uninitialize_delta_index(delta_index); + return result; + } + + delta_index->memory_size += + (sizeof(struct delta_zone) + zone_memory + + (lists_in_zone + 2) * (sizeof(struct delta_list) + sizeof(u64))); + } + + uds_reset_delta_index(delta_index); + return UDS_SUCCESS; +} + +/* Read a bit field from an arbitrary bit boundary. */ +static inline u32 get_field(const u8 *memory, u64 offset, u8 size) +{ + const void *addr = memory + offset / BITS_PER_BYTE; + + return (get_unaligned_le32(addr) >> (offset % BITS_PER_BYTE)) & ((1 << size) - 1); +} + +/* Write a bit field to an arbitrary bit boundary. */ +static inline void set_field(u32 value, u8 *memory, u64 offset, u8 size) +{ + void *addr = memory + offset / BITS_PER_BYTE; + int shift = offset % BITS_PER_BYTE; + u32 data = get_unaligned_le32(addr); + + data &= ~(((1 << size) - 1) << shift); + data |= value << shift; + put_unaligned_le32(data, addr); +} + +/* Get the bit offset to the immutable delta list header. */ +static inline u32 get_immutable_header_offset(u32 list_number) +{ + return sizeof(struct delta_page_header) * BITS_PER_BYTE + + list_number * IMMUTABLE_HEADER_SIZE; +} + +/* Get the bit offset to the start of the immutable delta list bit stream. */ +static inline u32 get_immutable_start(const u8 *memory, u32 list_number) +{ + return get_field(memory, get_immutable_header_offset(list_number), + IMMUTABLE_HEADER_SIZE); +} + +/* Set the bit offset to the start of the immutable delta list bit stream. */ +static inline void set_immutable_start(u8 *memory, u32 list_number, u32 start) +{ + set_field(start, memory, get_immutable_header_offset(list_number), + IMMUTABLE_HEADER_SIZE); +} + +static bool verify_delta_index_page(u64 nonce, u16 list_count, u64 expected_nonce, + u8 *memory, size_t memory_size) +{ + unsigned int i; + + /* + * Verify the nonce. A mismatch can happen here during rebuild if we haven't written the + * entire volume at least once. + */ + if (nonce != expected_nonce) + return false; + + /* Verify that the number of delta lists can fit in the page. */ + if (list_count > ((memory_size - sizeof(struct delta_page_header)) * + BITS_PER_BYTE / IMMUTABLE_HEADER_SIZE)) + return false; + + /* + * Verify that the first delta list is immediately after the last delta + * list header. + */ + if (get_immutable_start(memory, 0) != get_immutable_header_offset(list_count + 1)) + return false; + + /* Verify that the lists are in the correct order. */ + for (i = 0; i < list_count; i++) { + if (get_immutable_start(memory, i) > get_immutable_start(memory, i + 1)) + return false; + } + + /* + * Verify that the last list ends on the page, and that there is room + * for the post-field guard bits. + */ + if (get_immutable_start(memory, list_count) > + (memory_size - POST_FIELD_GUARD_BYTES) * BITS_PER_BYTE) + return false; + + /* Verify that the guard bytes are correctly set to all ones. */ + for (i = 0; i < POST_FIELD_GUARD_BYTES; i++) { + if (memory[memory_size - POST_FIELD_GUARD_BYTES + i] != (u8) ~0) + return false; + } + + /* All verifications passed. */ + return true; +} + +/* Initialize a delta index page to refer to a supplied page. */ +int uds_initialize_delta_index_page(struct delta_index_page *delta_index_page, + u64 expected_nonce, u32 mean_delta, u32 payload_bits, + u8 *memory, size_t memory_size) +{ + u64 nonce; + u64 vcn; + u64 first_list; + u64 list_count; + struct delta_page_header *header = (struct delta_page_header *) memory; + struct delta_zone *delta_zone = &delta_index_page->delta_zone; + const u8 *nonce_addr = (const u8 *) &header->nonce; + const u8 *vcn_addr = (const u8 *) &header->virtual_chapter_number; + const u8 *first_list_addr = (const u8 *) &header->first_list; + const u8 *list_count_addr = (const u8 *) &header->list_count; + + /* First assume that the header is little endian. */ + nonce = get_unaligned_le64(nonce_addr); + vcn = get_unaligned_le64(vcn_addr); + first_list = get_unaligned_le16(first_list_addr); + list_count = get_unaligned_le16(list_count_addr); + if (!verify_delta_index_page(nonce, list_count, expected_nonce, memory, + memory_size)) { + /* If that fails, try big endian. */ + nonce = get_unaligned_be64(nonce_addr); + vcn = get_unaligned_be64(vcn_addr); + first_list = get_unaligned_be16(first_list_addr); + list_count = get_unaligned_be16(list_count_addr); + if (!verify_delta_index_page(nonce, list_count, expected_nonce, memory, + memory_size)) { + /* + * Both attempts failed. Do not log this as an error, because it can happen + * during a rebuild if we haven't written the entire volume at least once. + */ + return UDS_CORRUPT_DATA; + } + } + + delta_index_page->delta_index.delta_zones = delta_zone; + delta_index_page->delta_index.zone_count = 1; + delta_index_page->delta_index.list_count = list_count; + delta_index_page->delta_index.lists_per_zone = list_count; + delta_index_page->delta_index.mutable = false; + delta_index_page->delta_index.tag = 'p'; + delta_index_page->virtual_chapter_number = vcn; + delta_index_page->lowest_list_number = first_list; + delta_index_page->highest_list_number = first_list + list_count - 1; + + compute_coding_constants(mean_delta, &delta_zone->min_bits, + &delta_zone->min_keys, &delta_zone->incr_keys); + delta_zone->value_bits = payload_bits; + delta_zone->memory = memory; + delta_zone->delta_lists = NULL; + delta_zone->new_offsets = NULL; + delta_zone->buffered_writer = NULL; + delta_zone->size = memory_size; + delta_zone->rebalance_time = 0; + delta_zone->rebalance_count = 0; + delta_zone->record_count = 0; + delta_zone->collision_count = 0; + delta_zone->discard_count = 0; + delta_zone->overflow_count = 0; + delta_zone->first_list = 0; + delta_zone->list_count = list_count; + delta_zone->tag = 'p'; + + return UDS_SUCCESS; +} + +/* Read a large bit field from an arbitrary bit boundary. */ +static inline u64 get_big_field(const u8 *memory, u64 offset, u8 size) +{ + const void *addr = memory + offset / BITS_PER_BYTE; + + return (get_unaligned_le64(addr) >> (offset % BITS_PER_BYTE)) & ((1UL << size) - 1); +} + +/* Write a large bit field to an arbitrary bit boundary. */ +static inline void set_big_field(u64 value, u8 *memory, u64 offset, u8 size) +{ + void *addr = memory + offset / BITS_PER_BYTE; + u8 shift = offset % BITS_PER_BYTE; + u64 data = get_unaligned_le64(addr); + + data &= ~(((1UL << size) - 1) << shift); + data |= value << shift; + put_unaligned_le64(data, addr); +} + +/* Set a sequence of bits to all zeros. */ +static inline void set_zero(u8 *memory, u64 offset, u32 size) +{ + if (size > 0) { + u8 *addr = memory + offset / BITS_PER_BYTE; + u8 shift = offset % BITS_PER_BYTE; + u32 count = size + shift > BITS_PER_BYTE ? (u32) BITS_PER_BYTE - shift : size; + + *addr++ &= ~(((1 << count) - 1) << shift); + for (size -= count; size > BITS_PER_BYTE; size -= BITS_PER_BYTE) + *addr++ = 0; + + if (size > 0) + *addr &= 0xFF << size; + } +} + +/* + * Move several bits from a higher to a lower address, moving the lower addressed bits first. The + * size and memory offsets are measured in bits. + */ +static void move_bits_down(const u8 *from, u64 from_offset, u8 *to, u64 to_offset, u32 size) +{ + const u8 *source; + u8 *destination; + u8 offset; + u8 count; + u64 field; + + /* Start by moving one field that ends on a to int boundary. */ + count = (MAX_BIG_FIELD_BITS - ((to_offset + MAX_BIG_FIELD_BITS) % BITS_PER_TYPE(u32))); + field = get_big_field(from, from_offset, count); + set_big_field(field, to, to_offset, count); + from_offset += count; + to_offset += count; + size -= count; + + /* Now do the main loop to copy 32 bit chunks that are int-aligned at the destination. */ + offset = from_offset % BITS_PER_TYPE(u32); + source = from + (from_offset - offset) / BITS_PER_BYTE; + destination = to + to_offset / BITS_PER_BYTE; + while (size > MAX_BIG_FIELD_BITS) { + put_unaligned_le32(get_unaligned_le64(source) >> offset, destination); + source += sizeof(u32); + destination += sizeof(u32); + from_offset += BITS_PER_TYPE(u32); + to_offset += BITS_PER_TYPE(u32); + size -= BITS_PER_TYPE(u32); + } + + /* Finish up by moving any remaining bits. */ + if (size > 0) { + field = get_big_field(from, from_offset, size); + set_big_field(field, to, to_offset, size); + } +} + +/* + * Move several bits from a lower to a higher address, moving the higher addressed bits first. The + * size and memory offsets are measured in bits. + */ +static void move_bits_up(const u8 *from, u64 from_offset, u8 *to, u64 to_offset, u32 size) +{ + const u8 *source; + u8 *destination; + u8 offset; + u8 count; + u64 field; + + /* Start by moving one field that begins on a destination int boundary. */ + count = (to_offset + size) % BITS_PER_TYPE(u32); + if (count > 0) { + size -= count; + field = get_big_field(from, from_offset + size, count); + set_big_field(field, to, to_offset + size, count); + } + + /* Now do the main loop to copy 32 bit chunks that are int-aligned at the destination. */ + offset = (from_offset + size) % BITS_PER_TYPE(u32); + source = from + (from_offset + size - offset) / BITS_PER_BYTE; + destination = to + (to_offset + size) / BITS_PER_BYTE; + while (size > MAX_BIG_FIELD_BITS) { + source -= sizeof(u32); + destination -= sizeof(u32); + size -= BITS_PER_TYPE(u32); + put_unaligned_le32(get_unaligned_le64(source) >> offset, destination); + } + + /* Finish up by moving any remaining bits. */ + if (size > 0) { + field = get_big_field(from, from_offset, size); + set_big_field(field, to, to_offset, size); + } +} + +/* + * Move bits from one field to another. When the fields overlap, behave as if we first move all the + * bits from the source to a temporary value, and then move all the bits from the temporary value + * to the destination. The size and memory offsets are measured in bits. + */ +static void move_bits(const u8 *from, u64 from_offset, u8 *to, u64 to_offset, u32 size) +{ + u64 field; + + /* A small move doesn't require special handling. */ + if (size <= MAX_BIG_FIELD_BITS) { + if (size > 0) { + field = get_big_field(from, from_offset, size); + set_big_field(field, to, to_offset, size); + } + + return; + } + + if (from_offset > to_offset) + move_bits_down(from, from_offset, to, to_offset, size); + else + move_bits_up(from, from_offset, to, to_offset, size); +} + +/* + * Pack delta lists from a mutable delta index into an immutable delta index page. A range of delta + * lists (starting with a specified list index) is copied from the mutable delta index into a + * memory page used in the immutable index. The number of lists copied onto the page is returned in + * list_count. + */ +int uds_pack_delta_index_page(const struct delta_index *delta_index, u64 header_nonce, + u8 *memory, size_t memory_size, u64 virtual_chapter_number, + u32 first_list, u32 *list_count) +{ + const struct delta_zone *delta_zone; + struct delta_list *delta_lists; + u32 max_lists; + u32 n_lists = 0; + u32 offset; + u32 i; + int free_bits; + int bits; + struct delta_page_header *header; + + delta_zone = &delta_index->delta_zones[0]; + delta_lists = &delta_zone->delta_lists[first_list + 1]; + max_lists = delta_index->list_count - first_list; + + /* + * Compute how many lists will fit on the page. Subtract the size of the fixed header, one + * delta list offset, and the guard bytes from the page size to determine how much space is + * available for delta lists. + */ + free_bits = memory_size * BITS_PER_BYTE; + free_bits -= get_immutable_header_offset(1); + free_bits -= GUARD_BITS; + if (free_bits < IMMUTABLE_HEADER_SIZE) { + /* This page is too small to store any delta lists. */ + return vdo_log_error_strerror(UDS_OVERFLOW, + "Chapter Index Page of %zu bytes is too small", + memory_size); + } + + while (n_lists < max_lists) { + /* Each list requires a delta list offset and the list data. */ + bits = IMMUTABLE_HEADER_SIZE + delta_lists[n_lists].size; + if (bits > free_bits) + break; + + n_lists++; + free_bits -= bits; + } + + *list_count = n_lists; + + header = (struct delta_page_header *) memory; + put_unaligned_le64(header_nonce, (u8 *) &header->nonce); + put_unaligned_le64(virtual_chapter_number, + (u8 *) &header->virtual_chapter_number); + put_unaligned_le16(first_list, (u8 *) &header->first_list); + put_unaligned_le16(n_lists, (u8 *) &header->list_count); + + /* Construct the delta list offset table. */ + offset = get_immutable_header_offset(n_lists + 1); + set_immutable_start(memory, 0, offset); + for (i = 0; i < n_lists; i++) { + offset += delta_lists[i].size; + set_immutable_start(memory, i + 1, offset); + } + + /* Copy the delta list data onto the memory page. */ + for (i = 0; i < n_lists; i++) { + move_bits(delta_zone->memory, delta_lists[i].start, memory, + get_immutable_start(memory, i), delta_lists[i].size); + } + + /* Set all the bits in the guard bytes. */ + memset(memory + memory_size - POST_FIELD_GUARD_BYTES, ~0, + POST_FIELD_GUARD_BYTES); + return UDS_SUCCESS; +} + +/* Compute the new offsets of the delta lists. */ +static void compute_new_list_offsets(struct delta_zone *delta_zone, u32 growing_index, + size_t growing_size, size_t used_space) +{ + size_t spacing; + u32 i; + struct delta_list *delta_lists = delta_zone->delta_lists; + u32 tail_guard_index = delta_zone->list_count + 1; + + spacing = (delta_zone->size - used_space) / delta_zone->list_count; + delta_zone->new_offsets[0] = 0; + for (i = 0; i <= delta_zone->list_count; i++) { + delta_zone->new_offsets[i + 1] = + (delta_zone->new_offsets[i] + + get_delta_list_byte_size(&delta_lists[i]) + spacing); + delta_zone->new_offsets[i] *= BITS_PER_BYTE; + delta_zone->new_offsets[i] += delta_lists[i].start % BITS_PER_BYTE; + if (i == 0) + delta_zone->new_offsets[i + 1] -= spacing / 2; + if (i + 1 == growing_index) + delta_zone->new_offsets[i + 1] += growing_size; + } + + delta_zone->new_offsets[tail_guard_index] = + (delta_zone->size * BITS_PER_BYTE - delta_lists[tail_guard_index].size); +} + +static void rebalance_lists(struct delta_zone *delta_zone) +{ + struct delta_list *delta_lists; + u32 i; + size_t used_space = 0; + + /* Extend and balance memory to receive the delta lists */ + delta_lists = delta_zone->delta_lists; + for (i = 0; i <= delta_zone->list_count + 1; i++) + used_space += get_delta_list_byte_size(&delta_lists[i]); + + compute_new_list_offsets(delta_zone, 0, 0, used_space); + for (i = 1; i <= delta_zone->list_count + 1; i++) + delta_lists[i].start = delta_zone->new_offsets[i]; +} + +/* Start restoring a delta index from multiple input streams. */ +int uds_start_restoring_delta_index(struct delta_index *delta_index, + struct buffered_reader **buffered_readers, + unsigned int reader_count) +{ + int result; + unsigned int zone_count = reader_count; + u64 record_count = 0; + u64 collision_count = 0; + u32 first_list[MAX_ZONES]; + u32 list_count[MAX_ZONES]; + unsigned int z; + u32 list_next = 0; + const struct delta_zone *delta_zone; + + /* Read and validate each header. */ + for (z = 0; z < zone_count; z++) { + struct delta_index_header header; + u8 buffer[sizeof(struct delta_index_header)]; + size_t offset = 0; + + result = uds_read_from_buffered_reader(buffered_readers[z], buffer, + sizeof(buffer)); + if (result != UDS_SUCCESS) { + return vdo_log_warning_strerror(result, + "failed to read delta index header"); + } + + memcpy(&header.magic, buffer, MAGIC_SIZE); + offset += MAGIC_SIZE; + decode_u32_le(buffer, &offset, &header.zone_number); + decode_u32_le(buffer, &offset, &header.zone_count); + decode_u32_le(buffer, &offset, &header.first_list); + decode_u32_le(buffer, &offset, &header.list_count); + decode_u64_le(buffer, &offset, &header.record_count); + decode_u64_le(buffer, &offset, &header.collision_count); + + result = VDO_ASSERT(offset == sizeof(struct delta_index_header), + "%zu bytes decoded of %zu expected", offset, + sizeof(struct delta_index_header)); + if (result != VDO_SUCCESS) { + return vdo_log_warning_strerror(result, + "failed to read delta index header"); + } + + if (memcmp(header.magic, DELTA_INDEX_MAGIC, MAGIC_SIZE) != 0) { + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, + "delta index file has bad magic number"); + } + + if (zone_count != header.zone_count) { + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, + "delta index files contain mismatched zone counts (%u,%u)", + zone_count, header.zone_count); + } + + if (header.zone_number != z) { + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, + "delta index zone %u found in slot %u", + header.zone_number, z); + } + + first_list[z] = header.first_list; + list_count[z] = header.list_count; + record_count += header.record_count; + collision_count += header.collision_count; + + if (first_list[z] != list_next) { + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, + "delta index file for zone %u starts with list %u instead of list %u", + z, first_list[z], list_next); + } + + list_next += list_count[z]; + } + + if (list_next != delta_index->list_count) { + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, + "delta index files contain %u delta lists instead of %u delta lists", + list_next, delta_index->list_count); + } + + if (collision_count > record_count) { + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, + "delta index files contain %llu collisions and %llu records", + (unsigned long long) collision_count, + (unsigned long long) record_count); + } + + uds_reset_delta_index(delta_index); + delta_index->delta_zones[0].record_count = record_count; + delta_index->delta_zones[0].collision_count = collision_count; + + /* Read the delta lists and distribute them to the proper zones. */ + for (z = 0; z < zone_count; z++) { + u32 i; + + delta_index->load_lists[z] = 0; + for (i = 0; i < list_count[z]; i++) { + u16 delta_list_size; + u32 list_number; + unsigned int zone_number; + u8 size_data[sizeof(u16)]; + + result = uds_read_from_buffered_reader(buffered_readers[z], + size_data, + sizeof(size_data)); + if (result != UDS_SUCCESS) { + return vdo_log_warning_strerror(result, + "failed to read delta index size"); + } + + delta_list_size = get_unaligned_le16(size_data); + if (delta_list_size > 0) + delta_index->load_lists[z] += 1; + + list_number = first_list[z] + i; + zone_number = list_number / delta_index->lists_per_zone; + delta_zone = &delta_index->delta_zones[zone_number]; + list_number -= delta_zone->first_list; + delta_zone->delta_lists[list_number + 1].size = delta_list_size; + } + } + + /* Prepare each zone to start receiving the delta list data. */ + for (z = 0; z < delta_index->zone_count; z++) + rebalance_lists(&delta_index->delta_zones[z]); + + return UDS_SUCCESS; +} + +static int restore_delta_list_to_zone(struct delta_zone *delta_zone, + const struct delta_list_save_info *save_info, + const u8 *data) +{ + struct delta_list *delta_list; + u16 bit_count; + u16 byte_count; + u32 list_number = save_info->index - delta_zone->first_list; + + if (list_number >= delta_zone->list_count) { + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, + "invalid delta list number %u not in range [%u,%u)", + save_info->index, delta_zone->first_list, + delta_zone->first_list + delta_zone->list_count); + } + + delta_list = &delta_zone->delta_lists[list_number + 1]; + if (delta_list->size == 0) { + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, + "unexpected delta list number %u", + save_info->index); + } + + bit_count = delta_list->size + save_info->bit_offset; + byte_count = BITS_TO_BYTES(bit_count); + if (save_info->byte_count != byte_count) { + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, + "unexpected delta list size %u != %u", + save_info->byte_count, byte_count); + } + + move_bits(data, save_info->bit_offset, delta_zone->memory, delta_list->start, + delta_list->size); + return UDS_SUCCESS; +} + +static int restore_delta_list_data(struct delta_index *delta_index, unsigned int load_zone, + struct buffered_reader *buffered_reader, u8 *data) +{ + int result; + struct delta_list_save_info save_info; + u8 buffer[sizeof(struct delta_list_save_info)]; + unsigned int new_zone; + + result = uds_read_from_buffered_reader(buffered_reader, buffer, sizeof(buffer)); + if (result != UDS_SUCCESS) { + return vdo_log_warning_strerror(result, + "failed to read delta list data"); + } + + save_info = (struct delta_list_save_info) { + .tag = buffer[0], + .bit_offset = buffer[1], + .byte_count = get_unaligned_le16(&buffer[2]), + .index = get_unaligned_le32(&buffer[4]), + }; + + if ((save_info.bit_offset >= BITS_PER_BYTE) || + (save_info.byte_count > DELTA_LIST_MAX_BYTE_COUNT)) { + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, + "corrupt delta list data"); + } + + /* Make sure the data is intended for this delta index. */ + if (save_info.tag != delta_index->tag) + return UDS_CORRUPT_DATA; + + if (save_info.index >= delta_index->list_count) { + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, + "invalid delta list number %u of %u", + save_info.index, + delta_index->list_count); + } + + result = uds_read_from_buffered_reader(buffered_reader, data, + save_info.byte_count); + if (result != UDS_SUCCESS) { + return vdo_log_warning_strerror(result, + "failed to read delta list data"); + } + + delta_index->load_lists[load_zone] -= 1; + new_zone = save_info.index / delta_index->lists_per_zone; + return restore_delta_list_to_zone(&delta_index->delta_zones[new_zone], + &save_info, data); +} + +/* Restore delta lists from saved data. */ +int uds_finish_restoring_delta_index(struct delta_index *delta_index, + struct buffered_reader **buffered_readers, + unsigned int reader_count) +{ + int result; + int saved_result = UDS_SUCCESS; + unsigned int z; + u8 *data; + + result = vdo_allocate(DELTA_LIST_MAX_BYTE_COUNT, u8, __func__, &data); + if (result != VDO_SUCCESS) + return result; + + for (z = 0; z < reader_count; z++) { + while (delta_index->load_lists[z] > 0) { + result = restore_delta_list_data(delta_index, z, + buffered_readers[z], data); + if (result != UDS_SUCCESS) { + saved_result = result; + break; + } + } + } + + vdo_free(data); + return saved_result; +} + +int uds_check_guard_delta_lists(struct buffered_reader **buffered_readers, + unsigned int reader_count) +{ + int result; + unsigned int z; + u8 buffer[sizeof(struct delta_list_save_info)]; + + for (z = 0; z < reader_count; z++) { + result = uds_read_from_buffered_reader(buffered_readers[z], buffer, + sizeof(buffer)); + if (result != UDS_SUCCESS) + return result; + + if (buffer[0] != 'z') + return UDS_CORRUPT_DATA; + } + + return UDS_SUCCESS; +} + +static int flush_delta_list(struct delta_zone *zone, u32 flush_index) +{ + struct delta_list *delta_list; + u8 buffer[sizeof(struct delta_list_save_info)]; + int result; + + delta_list = &zone->delta_lists[flush_index + 1]; + + buffer[0] = zone->tag; + buffer[1] = delta_list->start % BITS_PER_BYTE; + put_unaligned_le16(get_delta_list_byte_size(delta_list), &buffer[2]); + put_unaligned_le32(zone->first_list + flush_index, &buffer[4]); + + result = uds_write_to_buffered_writer(zone->buffered_writer, buffer, + sizeof(buffer)); + if (result != UDS_SUCCESS) { + vdo_log_warning_strerror(result, "failed to write delta list memory"); + return result; + } + + result = uds_write_to_buffered_writer(zone->buffered_writer, + zone->memory + get_delta_list_byte_start(delta_list), + get_delta_list_byte_size(delta_list)); + if (result != UDS_SUCCESS) + vdo_log_warning_strerror(result, "failed to write delta list memory"); + + return result; +} + +/* Start saving a delta index zone to a buffered output stream. */ +int uds_start_saving_delta_index(const struct delta_index *delta_index, + unsigned int zone_number, + struct buffered_writer *buffered_writer) +{ + int result; + u32 i; + struct delta_zone *delta_zone; + u8 buffer[sizeof(struct delta_index_header)]; + size_t offset = 0; + + delta_zone = &delta_index->delta_zones[zone_number]; + memcpy(buffer, DELTA_INDEX_MAGIC, MAGIC_SIZE); + offset += MAGIC_SIZE; + encode_u32_le(buffer, &offset, zone_number); + encode_u32_le(buffer, &offset, delta_index->zone_count); + encode_u32_le(buffer, &offset, delta_zone->first_list); + encode_u32_le(buffer, &offset, delta_zone->list_count); + encode_u64_le(buffer, &offset, delta_zone->record_count); + encode_u64_le(buffer, &offset, delta_zone->collision_count); + + result = VDO_ASSERT(offset == sizeof(struct delta_index_header), + "%zu bytes encoded of %zu expected", offset, + sizeof(struct delta_index_header)); + if (result != VDO_SUCCESS) + return result; + + result = uds_write_to_buffered_writer(buffered_writer, buffer, offset); + if (result != UDS_SUCCESS) + return vdo_log_warning_strerror(result, + "failed to write delta index header"); + + for (i = 0; i < delta_zone->list_count; i++) { + u8 data[sizeof(u16)]; + struct delta_list *delta_list; + + delta_list = &delta_zone->delta_lists[i + 1]; + put_unaligned_le16(delta_list->size, data); + result = uds_write_to_buffered_writer(buffered_writer, data, + sizeof(data)); + if (result != UDS_SUCCESS) + return vdo_log_warning_strerror(result, + "failed to write delta list size"); + } + + delta_zone->buffered_writer = buffered_writer; + return UDS_SUCCESS; +} + +int uds_finish_saving_delta_index(const struct delta_index *delta_index, + unsigned int zone_number) +{ + int result; + int first_error = UDS_SUCCESS; + u32 i; + struct delta_zone *delta_zone; + struct delta_list *delta_list; + + delta_zone = &delta_index->delta_zones[zone_number]; + for (i = 0; i < delta_zone->list_count; i++) { + delta_list = &delta_zone->delta_lists[i + 1]; + if (delta_list->size > 0) { + result = flush_delta_list(delta_zone, i); + if ((result != UDS_SUCCESS) && (first_error == UDS_SUCCESS)) + first_error = result; + } + } + + delta_zone->buffered_writer = NULL; + return first_error; +} + +int uds_write_guard_delta_list(struct buffered_writer *buffered_writer) +{ + int result; + u8 buffer[sizeof(struct delta_list_save_info)]; + + memset(buffer, 0, sizeof(struct delta_list_save_info)); + buffer[0] = 'z'; + + result = uds_write_to_buffered_writer(buffered_writer, buffer, sizeof(buffer)); + if (result != UDS_SUCCESS) + vdo_log_warning_strerror(result, "failed to write guard delta list"); + + return UDS_SUCCESS; +} + +size_t uds_compute_delta_index_save_bytes(u32 list_count, size_t memory_size) +{ + /* One zone will use at least as much memory as other zone counts. */ + return (sizeof(struct delta_index_header) + + list_count * (sizeof(struct delta_list_save_info) + 1) + + get_zone_memory_size(1, memory_size)); +} + +static int assert_not_at_end(const struct delta_index_entry *delta_entry) +{ + int result = VDO_ASSERT(!delta_entry->at_end, + "operation is invalid because the list entry is at the end of the delta list"); + if (result != VDO_SUCCESS) + result = UDS_BAD_STATE; + + return result; +} + +/* + * Prepare to search for an entry in the specified delta list. + * + * This is always the first function to be called when dealing with delta index entries. It is + * always followed by calls to uds_next_delta_index_entry() to iterate through a delta list. The + * fields of the delta_index_entry argument will be set up for iteration, but will not contain an + * entry from the list. + */ +int uds_start_delta_index_search(const struct delta_index *delta_index, u32 list_number, + u32 key, struct delta_index_entry *delta_entry) +{ + int result; + unsigned int zone_number; + struct delta_zone *delta_zone; + struct delta_list *delta_list; + + result = VDO_ASSERT((list_number < delta_index->list_count), + "Delta list number (%u) is out of range (%u)", list_number, + delta_index->list_count); + if (result != VDO_SUCCESS) + return UDS_CORRUPT_DATA; + + zone_number = list_number / delta_index->lists_per_zone; + delta_zone = &delta_index->delta_zones[zone_number]; + list_number -= delta_zone->first_list; + result = VDO_ASSERT((list_number < delta_zone->list_count), + "Delta list number (%u) is out of range (%u) for zone (%u)", + list_number, delta_zone->list_count, zone_number); + if (result != VDO_SUCCESS) + return UDS_CORRUPT_DATA; + + if (delta_index->mutable) { + delta_list = &delta_zone->delta_lists[list_number + 1]; + } else { + u32 end_offset; + + /* + * Translate the immutable delta list header into a temporary + * full delta list header. + */ + delta_list = &delta_entry->temp_delta_list; + delta_list->start = get_immutable_start(delta_zone->memory, list_number); + end_offset = get_immutable_start(delta_zone->memory, list_number + 1); + delta_list->size = end_offset - delta_list->start; + delta_list->save_key = 0; + delta_list->save_offset = 0; + } + + if (key > delta_list->save_key) { + delta_entry->key = delta_list->save_key; + delta_entry->offset = delta_list->save_offset; + } else { + delta_entry->key = 0; + delta_entry->offset = 0; + if (key == 0) { + /* + * This usually means we're about to walk the entire delta list, so get all + * of it into the CPU cache. + */ + uds_prefetch_range(&delta_zone->memory[delta_list->start / BITS_PER_BYTE], + delta_list->size / BITS_PER_BYTE, false); + } + } + + delta_entry->at_end = false; + delta_entry->delta_zone = delta_zone; + delta_entry->delta_list = delta_list; + delta_entry->entry_bits = 0; + delta_entry->is_collision = false; + delta_entry->list_number = list_number; + delta_entry->list_overflow = false; + delta_entry->value_bits = delta_zone->value_bits; + return UDS_SUCCESS; +} + +static inline u64 get_delta_entry_offset(const struct delta_index_entry *delta_entry) +{ + return delta_entry->delta_list->start + delta_entry->offset; +} + +/* + * Decode a delta index entry delta value. The delta_index_entry basically describes the previous + * list entry, and has had its offset field changed to point to the subsequent entry. We decode the + * bit stream and update the delta_list_entry to describe the entry. + */ +static inline void decode_delta(struct delta_index_entry *delta_entry) +{ + int key_bits; + u32 delta; + const struct delta_zone *delta_zone = delta_entry->delta_zone; + const u8 *memory = delta_zone->memory; + u64 delta_offset = get_delta_entry_offset(delta_entry) + delta_entry->value_bits; + const u8 *addr = memory + delta_offset / BITS_PER_BYTE; + int offset = delta_offset % BITS_PER_BYTE; + u32 data = get_unaligned_le32(addr) >> offset; + + addr += sizeof(u32); + key_bits = delta_zone->min_bits; + delta = data & ((1 << key_bits) - 1); + if (delta >= delta_zone->min_keys) { + data >>= key_bits; + if (data == 0) { + key_bits = sizeof(u32) * BITS_PER_BYTE - offset; + while ((data = get_unaligned_le32(addr)) == 0) { + addr += sizeof(u32); + key_bits += sizeof(u32) * BITS_PER_BYTE; + } + } + key_bits += ffs(data); + delta += ((key_bits - delta_zone->min_bits - 1) * delta_zone->incr_keys); + } + delta_entry->delta = delta; + delta_entry->key += delta; + + /* Check for a collision, a delta of zero after the start. */ + if (unlikely((delta == 0) && (delta_entry->offset > 0))) { + delta_entry->is_collision = true; + delta_entry->entry_bits = delta_entry->value_bits + key_bits + COLLISION_BITS; + } else { + delta_entry->is_collision = false; + delta_entry->entry_bits = delta_entry->value_bits + key_bits; + } +} + +noinline int uds_next_delta_index_entry(struct delta_index_entry *delta_entry) +{ + int result; + const struct delta_list *delta_list; + u32 next_offset; + u16 size; + + result = assert_not_at_end(delta_entry); + if (result != UDS_SUCCESS) + return result; + + delta_list = delta_entry->delta_list; + delta_entry->offset += delta_entry->entry_bits; + size = delta_list->size; + if (unlikely(delta_entry->offset >= size)) { + delta_entry->at_end = true; + delta_entry->delta = 0; + delta_entry->is_collision = false; + result = VDO_ASSERT((delta_entry->offset == size), + "next offset past end of delta list"); + if (result != VDO_SUCCESS) + result = UDS_CORRUPT_DATA; + + return result; + } + + decode_delta(delta_entry); + + next_offset = delta_entry->offset + delta_entry->entry_bits; + if (next_offset > size) { + /* + * This is not an assertion because uds_validate_chapter_index_page() wants to + * handle this error. + */ + vdo_log_warning("Decoded past the end of the delta list"); + return UDS_CORRUPT_DATA; + } + + return UDS_SUCCESS; +} + +int uds_remember_delta_index_offset(const struct delta_index_entry *delta_entry) +{ + int result; + struct delta_list *delta_list = delta_entry->delta_list; + + result = VDO_ASSERT(!delta_entry->is_collision, "entry is not a collision"); + if (result != VDO_SUCCESS) + return result; + + delta_list->save_key = delta_entry->key - delta_entry->delta; + delta_list->save_offset = delta_entry->offset; + return UDS_SUCCESS; +} + +static void set_delta(struct delta_index_entry *delta_entry, u32 delta) +{ + const struct delta_zone *delta_zone = delta_entry->delta_zone; + u32 key_bits = (delta_zone->min_bits + + ((delta_zone->incr_keys - delta_zone->min_keys + delta) / + delta_zone->incr_keys)); + + delta_entry->delta = delta; + delta_entry->entry_bits = delta_entry->value_bits + key_bits; +} + +static void get_collision_name(const struct delta_index_entry *entry, u8 *name) +{ + u64 offset = get_delta_entry_offset(entry) + entry->entry_bits - COLLISION_BITS; + const u8 *addr = entry->delta_zone->memory + offset / BITS_PER_BYTE; + int size = COLLISION_BYTES; + int shift = offset % BITS_PER_BYTE; + + while (--size >= 0) + *name++ = get_unaligned_le16(addr++) >> shift; +} + +static void set_collision_name(const struct delta_index_entry *entry, const u8 *name) +{ + u64 offset = get_delta_entry_offset(entry) + entry->entry_bits - COLLISION_BITS; + u8 *addr = entry->delta_zone->memory + offset / BITS_PER_BYTE; + int size = COLLISION_BYTES; + int shift = offset % BITS_PER_BYTE; + u16 mask = ~((u16) 0xFF << shift); + u16 data; + + while (--size >= 0) { + data = (get_unaligned_le16(addr) & mask) | (*name++ << shift); + put_unaligned_le16(data, addr++); + } +} + +int uds_get_delta_index_entry(const struct delta_index *delta_index, u32 list_number, + u32 key, const u8 *name, + struct delta_index_entry *delta_entry) +{ + int result; + + result = uds_start_delta_index_search(delta_index, list_number, key, + delta_entry); + if (result != UDS_SUCCESS) + return result; + + do { + result = uds_next_delta_index_entry(delta_entry); + if (result != UDS_SUCCESS) + return result; + } while (!delta_entry->at_end && (key > delta_entry->key)); + + result = uds_remember_delta_index_offset(delta_entry); + if (result != UDS_SUCCESS) + return result; + + if (!delta_entry->at_end && (key == delta_entry->key)) { + struct delta_index_entry collision_entry = *delta_entry; + + for (;;) { + u8 full_name[COLLISION_BYTES]; + + result = uds_next_delta_index_entry(&collision_entry); + if (result != UDS_SUCCESS) + return result; + + if (collision_entry.at_end || !collision_entry.is_collision) + break; + + get_collision_name(&collision_entry, full_name); + if (memcmp(full_name, name, COLLISION_BYTES) == 0) { + *delta_entry = collision_entry; + break; + } + } + } + + return UDS_SUCCESS; +} + +int uds_get_delta_entry_collision(const struct delta_index_entry *delta_entry, u8 *name) +{ + int result; + + result = assert_not_at_end(delta_entry); + if (result != UDS_SUCCESS) + return result; + + result = VDO_ASSERT(delta_entry->is_collision, + "Cannot get full block name from a non-collision delta index entry"); + if (result != VDO_SUCCESS) + return UDS_BAD_STATE; + + get_collision_name(delta_entry, name); + return UDS_SUCCESS; +} + +u32 uds_get_delta_entry_value(const struct delta_index_entry *delta_entry) +{ + return get_field(delta_entry->delta_zone->memory, + get_delta_entry_offset(delta_entry), delta_entry->value_bits); +} + +static int assert_mutable_entry(const struct delta_index_entry *delta_entry) +{ + int result = VDO_ASSERT((delta_entry->delta_list != &delta_entry->temp_delta_list), + "delta index is mutable"); + if (result != VDO_SUCCESS) + result = UDS_BAD_STATE; + + return result; +} + +int uds_set_delta_entry_value(const struct delta_index_entry *delta_entry, u32 value) +{ + int result; + u32 value_mask = (1 << delta_entry->value_bits) - 1; + + result = assert_mutable_entry(delta_entry); + if (result != UDS_SUCCESS) + return result; + + result = assert_not_at_end(delta_entry); + if (result != UDS_SUCCESS) + return result; + + result = VDO_ASSERT((value & value_mask) == value, + "Value (%u) being set in a delta index is too large (must fit in %u bits)", + value, delta_entry->value_bits); + if (result != VDO_SUCCESS) + return UDS_INVALID_ARGUMENT; + + set_field(value, delta_entry->delta_zone->memory, + get_delta_entry_offset(delta_entry), delta_entry->value_bits); + return UDS_SUCCESS; +} + +/* + * Extend the memory used by the delta lists by adding growing_size bytes before the list indicated + * by growing_index, then rebalancing the lists in the new chunk. + */ +static int extend_delta_zone(struct delta_zone *delta_zone, u32 growing_index, + size_t growing_size) +{ + ktime_t start_time; + ktime_t end_time; + struct delta_list *delta_lists; + u32 i; + size_t used_space; + + + /* Calculate the amount of space that is or will be in use. */ + start_time = current_time_ns(CLOCK_MONOTONIC); + delta_lists = delta_zone->delta_lists; + used_space = growing_size; + for (i = 0; i <= delta_zone->list_count + 1; i++) + used_space += get_delta_list_byte_size(&delta_lists[i]); + + if (delta_zone->size < used_space) + return UDS_OVERFLOW; + + /* Compute the new offsets of the delta lists. */ + compute_new_list_offsets(delta_zone, growing_index, growing_size, used_space); + + /* + * When we rebalance the delta list, we will include the end guard list in the rebalancing. + * It contains the end guard data, which must be copied. + */ + rebalance_delta_zone(delta_zone, 1, delta_zone->list_count + 1); + end_time = current_time_ns(CLOCK_MONOTONIC); + delta_zone->rebalance_count++; + delta_zone->rebalance_time += ktime_sub(end_time, start_time); + return UDS_SUCCESS; +} + +static int insert_bits(struct delta_index_entry *delta_entry, u16 size) +{ + u64 free_before; + u64 free_after; + u64 source; + u64 destination; + u32 count; + bool before_flag; + u8 *memory; + struct delta_zone *delta_zone = delta_entry->delta_zone; + struct delta_list *delta_list = delta_entry->delta_list; + /* Compute bits in use before and after the inserted bits. */ + u32 total_size = delta_list->size; + u32 before_size = delta_entry->offset; + u32 after_size = total_size - delta_entry->offset; + + if (total_size + size > U16_MAX) { + delta_entry->list_overflow = true; + delta_zone->overflow_count++; + return UDS_OVERFLOW; + } + + /* Compute bits available before and after the delta list. */ + free_before = (delta_list[0].start - (delta_list[-1].start + delta_list[-1].size)); + free_after = (delta_list[1].start - (delta_list[0].start + delta_list[0].size)); + + if ((size <= free_before) && (size <= free_after)) { + /* + * We have enough space to use either before or after the list. Select the smaller + * amount of data. If it is exactly the same, try to take from the larger amount of + * free space. + */ + if (before_size < after_size) + before_flag = true; + else if (after_size < before_size) + before_flag = false; + else + before_flag = free_before > free_after; + } else if (size <= free_before) { + /* There is space before but not after. */ + before_flag = true; + } else if (size <= free_after) { + /* There is space after but not before. */ + before_flag = false; + } else { + /* + * Neither of the surrounding spaces is large enough for this request. Extend + * and/or rebalance the delta list memory choosing to move the least amount of + * data. + */ + int result; + u32 growing_index = delta_entry->list_number + 1; + + before_flag = before_size < after_size; + if (!before_flag) + growing_index++; + result = extend_delta_zone(delta_zone, growing_index, + BITS_TO_BYTES(size)); + if (result != UDS_SUCCESS) + return result; + } + + delta_list->size += size; + if (before_flag) { + source = delta_list->start; + destination = source - size; + delta_list->start -= size; + count = before_size; + } else { + source = delta_list->start + delta_entry->offset; + destination = source + size; + count = after_size; + } + + memory = delta_zone->memory; + move_bits(memory, source, memory, destination, count); + return UDS_SUCCESS; +} + +static void encode_delta(const struct delta_index_entry *delta_entry) +{ + u32 temp; + u32 t1; + u32 t2; + u64 offset; + const struct delta_zone *delta_zone = delta_entry->delta_zone; + u8 *memory = delta_zone->memory; + + offset = get_delta_entry_offset(delta_entry) + delta_entry->value_bits; + if (delta_entry->delta < delta_zone->min_keys) { + set_field(delta_entry->delta, memory, offset, delta_zone->min_bits); + return; + } + + temp = delta_entry->delta - delta_zone->min_keys; + t1 = (temp % delta_zone->incr_keys) + delta_zone->min_keys; + t2 = temp / delta_zone->incr_keys; + set_field(t1, memory, offset, delta_zone->min_bits); + set_zero(memory, offset + delta_zone->min_bits, t2); + set_field(1, memory, offset + delta_zone->min_bits + t2, 1); +} + +static void encode_entry(const struct delta_index_entry *delta_entry, u32 value, + const u8 *name) +{ + u8 *memory = delta_entry->delta_zone->memory; + u64 offset = get_delta_entry_offset(delta_entry); + + set_field(value, memory, offset, delta_entry->value_bits); + encode_delta(delta_entry); + if (name != NULL) + set_collision_name(delta_entry, name); +} + +/* + * Create a new entry in the delta index. If the entry is a collision, the full 256 bit name must + * be provided. + */ +int uds_put_delta_index_entry(struct delta_index_entry *delta_entry, u32 key, u32 value, + const u8 *name) +{ + int result; + struct delta_zone *delta_zone; + + result = assert_mutable_entry(delta_entry); + if (result != UDS_SUCCESS) + return result; + + if (delta_entry->is_collision) { + /* + * The caller wants us to insert a collision entry onto a collision entry. This + * happens when we find a collision and attempt to add the name again to the index. + * This is normally a fatal error unless we are replaying a closed chapter while we + * are rebuilding a volume index. + */ + return UDS_DUPLICATE_NAME; + } + + if (delta_entry->offset < delta_entry->delta_list->save_offset) { + /* + * The saved entry offset is after the new entry and will no longer be valid, so + * replace it with the insertion point. + */ + result = uds_remember_delta_index_offset(delta_entry); + if (result != UDS_SUCCESS) + return result; + } + + if (name != NULL) { + /* Insert a collision entry which is placed after this entry. */ + result = assert_not_at_end(delta_entry); + if (result != UDS_SUCCESS) + return result; + + result = VDO_ASSERT((key == delta_entry->key), + "incorrect key for collision entry"); + if (result != VDO_SUCCESS) + return result; + + delta_entry->offset += delta_entry->entry_bits; + set_delta(delta_entry, 0); + delta_entry->is_collision = true; + delta_entry->entry_bits += COLLISION_BITS; + result = insert_bits(delta_entry, delta_entry->entry_bits); + } else if (delta_entry->at_end) { + /* Insert a new entry at the end of the delta list. */ + result = VDO_ASSERT((key >= delta_entry->key), "key past end of list"); + if (result != VDO_SUCCESS) + return result; + + set_delta(delta_entry, key - delta_entry->key); + delta_entry->key = key; + delta_entry->at_end = false; + result = insert_bits(delta_entry, delta_entry->entry_bits); + } else { + u16 old_entry_size; + u16 additional_size; + struct delta_index_entry next_entry; + u32 next_value; + + /* + * Insert a new entry which requires the delta in the following entry to be + * updated. + */ + result = VDO_ASSERT((key < delta_entry->key), + "key precedes following entry"); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT((key >= delta_entry->key - delta_entry->delta), + "key effects following entry's delta"); + if (result != VDO_SUCCESS) + return result; + + old_entry_size = delta_entry->entry_bits; + next_entry = *delta_entry; + next_value = uds_get_delta_entry_value(&next_entry); + set_delta(delta_entry, key - (delta_entry->key - delta_entry->delta)); + delta_entry->key = key; + set_delta(&next_entry, next_entry.key - key); + next_entry.offset += delta_entry->entry_bits; + /* The two new entries are always bigger than the single entry being replaced. */ + additional_size = (delta_entry->entry_bits + + next_entry.entry_bits - old_entry_size); + result = insert_bits(delta_entry, additional_size); + if (result != UDS_SUCCESS) + return result; + + encode_entry(&next_entry, next_value, NULL); + } + + if (result != UDS_SUCCESS) + return result; + + encode_entry(delta_entry, value, name); + delta_zone = delta_entry->delta_zone; + delta_zone->record_count++; + delta_zone->collision_count += delta_entry->is_collision ? 1 : 0; + return UDS_SUCCESS; +} + +static void delete_bits(const struct delta_index_entry *delta_entry, int size) +{ + u64 source; + u64 destination; + u32 count; + bool before_flag; + struct delta_list *delta_list = delta_entry->delta_list; + u8 *memory = delta_entry->delta_zone->memory; + /* Compute bits retained before and after the deleted bits. */ + u32 total_size = delta_list->size; + u32 before_size = delta_entry->offset; + u32 after_size = total_size - delta_entry->offset - size; + + /* + * Determine whether to add to the available space either before or after the delta list. + * We prefer to move the least amount of data. If it is exactly the same, try to add to the + * smaller amount of free space. + */ + if (before_size < after_size) { + before_flag = true; + } else if (after_size < before_size) { + before_flag = false; + } else { + u64 free_before = + (delta_list[0].start - (delta_list[-1].start + delta_list[-1].size)); + u64 free_after = + (delta_list[1].start - (delta_list[0].start + delta_list[0].size)); + + before_flag = (free_before < free_after); + } + + delta_list->size -= size; + if (before_flag) { + source = delta_list->start; + destination = source + size; + delta_list->start += size; + count = before_size; + } else { + destination = delta_list->start + delta_entry->offset; + source = destination + size; + count = after_size; + } + + move_bits(memory, source, memory, destination, count); +} + +int uds_remove_delta_index_entry(struct delta_index_entry *delta_entry) +{ + int result; + struct delta_index_entry next_entry; + struct delta_zone *delta_zone; + struct delta_list *delta_list; + + result = assert_mutable_entry(delta_entry); + if (result != UDS_SUCCESS) + return result; + + next_entry = *delta_entry; + result = uds_next_delta_index_entry(&next_entry); + if (result != UDS_SUCCESS) + return result; + + delta_zone = delta_entry->delta_zone; + + if (delta_entry->is_collision) { + /* This is a collision entry, so just remove it. */ + delete_bits(delta_entry, delta_entry->entry_bits); + next_entry.offset = delta_entry->offset; + delta_zone->collision_count -= 1; + } else if (next_entry.at_end) { + /* This entry is at the end of the list, so just remove it. */ + delete_bits(delta_entry, delta_entry->entry_bits); + next_entry.key -= delta_entry->delta; + next_entry.offset = delta_entry->offset; + } else { + /* The delta in the next entry needs to be updated. */ + u32 next_value = uds_get_delta_entry_value(&next_entry); + u16 old_size = delta_entry->entry_bits + next_entry.entry_bits; + + if (next_entry.is_collision) { + next_entry.is_collision = false; + delta_zone->collision_count -= 1; + } + + set_delta(&next_entry, delta_entry->delta + next_entry.delta); + next_entry.offset = delta_entry->offset; + /* The one new entry is always smaller than the two entries being replaced. */ + delete_bits(delta_entry, old_size - next_entry.entry_bits); + encode_entry(&next_entry, next_value, NULL); + } + + delta_zone->record_count--; + delta_zone->discard_count++; + *delta_entry = next_entry; + + delta_list = delta_entry->delta_list; + if (delta_entry->offset < delta_list->save_offset) { + /* The saved entry offset is no longer valid. */ + delta_list->save_key = 0; + delta_list->save_offset = 0; + } + + return UDS_SUCCESS; +} + +void uds_get_delta_index_stats(const struct delta_index *delta_index, + struct delta_index_stats *stats) +{ + unsigned int z; + const struct delta_zone *delta_zone; + + memset(stats, 0, sizeof(struct delta_index_stats)); + for (z = 0; z < delta_index->zone_count; z++) { + delta_zone = &delta_index->delta_zones[z]; + stats->rebalance_time += delta_zone->rebalance_time; + stats->rebalance_count += delta_zone->rebalance_count; + stats->record_count += delta_zone->record_count; + stats->collision_count += delta_zone->collision_count; + stats->discard_count += delta_zone->discard_count; + stats->overflow_count += delta_zone->overflow_count; + stats->list_count += delta_zone->list_count; + } +} + +size_t uds_compute_delta_index_size(u32 entry_count, u32 mean_delta, u32 payload_bits) +{ + u16 min_bits; + u32 incr_keys; + u32 min_keys; + + compute_coding_constants(mean_delta, &min_bits, &min_keys, &incr_keys); + /* On average, each delta is encoded into about min_bits + 1.5 bits. */ + return entry_count * (payload_bits + min_bits + 1) + entry_count / 2; +} + +u32 uds_get_delta_index_page_count(u32 entry_count, u32 list_count, u32 mean_delta, + u32 payload_bits, size_t bytes_per_page) +{ + unsigned int bits_per_delta_list; + unsigned int bits_per_page; + size_t bits_per_index; + + /* Compute the expected number of bits needed for all the entries. */ + bits_per_index = uds_compute_delta_index_size(entry_count, mean_delta, + payload_bits); + bits_per_delta_list = bits_per_index / list_count; + + /* Add in the immutable delta list headers. */ + bits_per_index += list_count * IMMUTABLE_HEADER_SIZE; + /* Compute the number of usable bits on an immutable index page. */ + bits_per_page = ((bytes_per_page - sizeof(struct delta_page_header)) * BITS_PER_BYTE); + /* + * Reduce the bits per page by one immutable delta list header and one delta list to + * account for internal fragmentation. + */ + bits_per_page -= IMMUTABLE_HEADER_SIZE + bits_per_delta_list; + /* Now compute the number of pages needed. */ + return DIV_ROUND_UP(bits_per_index, bits_per_page); +} + +void uds_log_delta_index_entry(struct delta_index_entry *delta_entry) +{ + vdo_log_ratelimit(vdo_log_info, + "List 0x%X Key 0x%X Offset 0x%X%s%s List_size 0x%X%s", + delta_entry->list_number, delta_entry->key, + delta_entry->offset, delta_entry->at_end ? " end" : "", + delta_entry->is_collision ? " collision" : "", + delta_entry->delta_list->size, + delta_entry->list_overflow ? " overflow" : ""); + delta_entry->list_overflow = false; +} diff --git a/drivers/md/dm-vdo/indexer/delta-index.h b/drivers/md/dm-vdo/indexer/delta-index.h new file mode 100644 index 000000000000..53f6c6ac0bc7 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/delta-index.h @@ -0,0 +1,279 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_DELTA_INDEX_H +#define UDS_DELTA_INDEX_H + +#include + +#include "numeric.h" +#include "time-utils.h" + +#include "config.h" +#include "io-factory.h" + +/* + * A delta index is a key-value store, where each entry maps an address (the key) to a payload (the + * value). The entries are sorted by address, and only the delta between successive addresses is + * stored in the entry. The addresses are assumed to be uniformly distributed, and the deltas are + * therefore exponentially distributed. + * + * A delta_index can either be mutable or immutable depending on its expected use. The immutable + * form of a delta index is used for the indexes of closed chapters committed to the volume. The + * mutable form of a delta index is used by the volume index, and also by the chapter index in an + * open chapter. Like the index as a whole, each mutable delta index is divided into a number of + * independent zones. + */ + +struct delta_list { + /* The offset of the delta list start, in bits */ + u64 start; + /* The number of bits in the delta list */ + u16 size; + /* Where the last search "found" the key, in bits */ + u16 save_offset; + /* The key for the record just before save_offset */ + u32 save_key; +}; + +struct delta_zone { + /* The delta list memory */ + u8 *memory; + /* The delta list headers */ + struct delta_list *delta_lists; + /* Temporary starts of delta lists */ + u64 *new_offsets; + /* Buffered writer for saving an index */ + struct buffered_writer *buffered_writer; + /* The size of delta list memory */ + size_t size; + /* Nanoseconds spent rebalancing */ + ktime_t rebalance_time; + /* Number of memory rebalances */ + u32 rebalance_count; + /* The number of bits in a stored value */ + u8 value_bits; + /* The number of bits in the minimal key code */ + u16 min_bits; + /* The number of keys used in a minimal code */ + u32 min_keys; + /* The number of keys used for another code bit */ + u32 incr_keys; + /* The number of records in the index */ + u64 record_count; + /* The number of collision records */ + u64 collision_count; + /* The number of records removed */ + u64 discard_count; + /* The number of UDS_OVERFLOW errors detected */ + u64 overflow_count; + /* The index of the first delta list */ + u32 first_list; + /* The number of delta lists */ + u32 list_count; + /* Tag belonging to this delta index */ + u8 tag; +} __aligned(L1_CACHE_BYTES); + +struct delta_list_save_info { + /* Tag identifying which delta index this list is in */ + u8 tag; + /* Bit offset of the start of the list data */ + u8 bit_offset; + /* Number of bytes of list data */ + u16 byte_count; + /* The delta list number within the delta index */ + u32 index; +} __packed; + +struct delta_index { + /* The zones */ + struct delta_zone *delta_zones; + /* The number of zones */ + unsigned int zone_count; + /* The number of delta lists */ + u32 list_count; + /* Maximum lists per zone */ + u32 lists_per_zone; + /* Total memory allocated to this index */ + size_t memory_size; + /* The number of non-empty lists at load time per zone */ + u32 load_lists[MAX_ZONES]; + /* True if this index is mutable */ + bool mutable; + /* Tag belonging to this delta index */ + u8 tag; +}; + +/* + * A delta_index_page describes a single page of a chapter index. The delta_index field allows the + * page to be treated as an immutable delta_index. We use the delta_zone field to treat the chapter + * index page as a single zone index, and without the need to do an additional memory allocation. + */ +struct delta_index_page { + struct delta_index delta_index; + /* These values are loaded from the delta_page_header */ + u32 lowest_list_number; + u32 highest_list_number; + u64 virtual_chapter_number; + /* This structure describes the single zone of a delta index page. */ + struct delta_zone delta_zone; +}; + +/* + * Notes on the delta_index_entries: + * + * The fields documented as "public" can be read by any code that uses a delta_index. The fields + * documented as "private" carry information between delta_index method calls and should not be + * used outside the delta_index module. + * + * (1) The delta_index_entry is used like an iterator when searching a delta list. + * + * (2) It is also the result of a successful search and can be used to refer to the element found + * by the search. + * + * (3) It is also the result of an unsuccessful search and can be used to refer to the insertion + * point for a new record. + * + * (4) If at_end is true, the delta_list entry can only be used as the insertion point for a new + * record at the end of the list. + * + * (5) If at_end is false and is_collision is true, the delta_list entry fields refer to a + * collision entry in the list, and the delta_list entry can be used as a reference to this + * entry. + * + * (6) If at_end is false and is_collision is false, the delta_list entry fields refer to a + * non-collision entry in the list. Such delta_list entries can be used as a reference to a + * found entry, or an insertion point for a non-collision entry before this entry, or an + * insertion point for a collision entry that collides with this entry. + */ +struct delta_index_entry { + /* Public fields */ + /* The key for this entry */ + u32 key; + /* We are after the last list entry */ + bool at_end; + /* This record is a collision */ + bool is_collision; + + /* Private fields */ + /* This delta list overflowed */ + bool list_overflow; + /* The number of bits used for the value */ + u8 value_bits; + /* The number of bits used for the entire entry */ + u16 entry_bits; + /* The delta index zone */ + struct delta_zone *delta_zone; + /* The delta list containing the entry */ + struct delta_list *delta_list; + /* The delta list number */ + u32 list_number; + /* Bit offset of this entry within the list */ + u16 offset; + /* The delta between this and previous entry */ + u32 delta; + /* Temporary delta list for immutable indices */ + struct delta_list temp_delta_list; +}; + +struct delta_index_stats { + /* Number of bytes allocated */ + size_t memory_allocated; + /* Nanoseconds spent rebalancing */ + ktime_t rebalance_time; + /* Number of memory rebalances */ + u32 rebalance_count; + /* The number of records in the index */ + u64 record_count; + /* The number of collision records */ + u64 collision_count; + /* The number of records removed */ + u64 discard_count; + /* The number of UDS_OVERFLOW errors detected */ + u64 overflow_count; + /* The number of delta lists */ + u32 list_count; +}; + +int __must_check uds_initialize_delta_index(struct delta_index *delta_index, + unsigned int zone_count, u32 list_count, + u32 mean_delta, u32 payload_bits, + size_t memory_size, u8 tag); + +int __must_check uds_initialize_delta_index_page(struct delta_index_page *delta_index_page, + u64 expected_nonce, u32 mean_delta, + u32 payload_bits, u8 *memory, + size_t memory_size); + +void uds_uninitialize_delta_index(struct delta_index *delta_index); + +void uds_reset_delta_index(const struct delta_index *delta_index); + +int __must_check uds_pack_delta_index_page(const struct delta_index *delta_index, + u64 header_nonce, u8 *memory, + size_t memory_size, + u64 virtual_chapter_number, u32 first_list, + u32 *list_count); + +int __must_check uds_start_restoring_delta_index(struct delta_index *delta_index, + struct buffered_reader **buffered_readers, + unsigned int reader_count); + +int __must_check uds_finish_restoring_delta_index(struct delta_index *delta_index, + struct buffered_reader **buffered_readers, + unsigned int reader_count); + +int __must_check uds_check_guard_delta_lists(struct buffered_reader **buffered_readers, + unsigned int reader_count); + +int __must_check uds_start_saving_delta_index(const struct delta_index *delta_index, + unsigned int zone_number, + struct buffered_writer *buffered_writer); + +int __must_check uds_finish_saving_delta_index(const struct delta_index *delta_index, + unsigned int zone_number); + +int __must_check uds_write_guard_delta_list(struct buffered_writer *buffered_writer); + +size_t __must_check uds_compute_delta_index_save_bytes(u32 list_count, + size_t memory_size); + +int __must_check uds_start_delta_index_search(const struct delta_index *delta_index, + u32 list_number, u32 key, + struct delta_index_entry *iterator); + +int __must_check uds_next_delta_index_entry(struct delta_index_entry *delta_entry); + +int __must_check uds_remember_delta_index_offset(const struct delta_index_entry *delta_entry); + +int __must_check uds_get_delta_index_entry(const struct delta_index *delta_index, + u32 list_number, u32 key, const u8 *name, + struct delta_index_entry *delta_entry); + +int __must_check uds_get_delta_entry_collision(const struct delta_index_entry *delta_entry, + u8 *name); + +u32 __must_check uds_get_delta_entry_value(const struct delta_index_entry *delta_entry); + +int __must_check uds_set_delta_entry_value(const struct delta_index_entry *delta_entry, u32 value); + +int __must_check uds_put_delta_index_entry(struct delta_index_entry *delta_entry, u32 key, + u32 value, const u8 *name); + +int __must_check uds_remove_delta_index_entry(struct delta_index_entry *delta_entry); + +void uds_get_delta_index_stats(const struct delta_index *delta_index, + struct delta_index_stats *stats); + +size_t __must_check uds_compute_delta_index_size(u32 entry_count, u32 mean_delta, + u32 payload_bits); + +u32 uds_get_delta_index_page_count(u32 entry_count, u32 list_count, u32 mean_delta, + u32 payload_bits, size_t bytes_per_page); + +void uds_log_delta_index_entry(struct delta_index_entry *delta_entry); + +#endif /* UDS_DELTA_INDEX_H */ diff --git a/drivers/md/dm-vdo/indexer/funnel-requestqueue.c b/drivers/md/dm-vdo/indexer/funnel-requestqueue.c new file mode 100644 index 000000000000..1a5735375ddc --- /dev/null +++ b/drivers/md/dm-vdo/indexer/funnel-requestqueue.c @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "funnel-requestqueue.h" + +#include +#include +#include + +#include "funnel-queue.h" +#include "logger.h" +#include "memory-alloc.h" +#include "thread-utils.h" + +/* + * This queue will attempt to handle requests in reasonably sized batches instead of reacting + * immediately to each new request. The wait time between batches is dynamically adjusted up or + * down to try to balance responsiveness against wasted thread run time. + * + * If the wait time becomes long enough, the queue will become dormant and must be explicitly + * awoken when a new request is enqueued. The enqueue operation updates "newest" in the funnel + * queue via xchg (which is a memory barrier), and later checks "dormant" to decide whether to do a + * wakeup of the worker thread. + * + * When deciding to go to sleep, the worker thread sets "dormant" and then examines "newest" to + * decide if the funnel queue is idle. In dormant mode, the last examination of "newest" before + * going to sleep is done inside the wait_event_interruptible() macro, after a point where one or + * more memory barriers have been issued. (Preparing to sleep uses spin locks.) Even if the funnel + * queue's "next" field update isn't visible yet to make the entry accessible, its existence will + * kick the worker thread out of dormant mode and back into timer-based mode. + * + * Unbatched requests are used to communicate between different zone threads and will also cause + * the queue to awaken immediately. + */ + +enum { + NANOSECOND = 1, + MICROSECOND = 1000 * NANOSECOND, + MILLISECOND = 1000 * MICROSECOND, + DEFAULT_WAIT_TIME = 20 * MICROSECOND, + MINIMUM_WAIT_TIME = DEFAULT_WAIT_TIME / 2, + MAXIMUM_WAIT_TIME = MILLISECOND, + MINIMUM_BATCH = 32, + MAXIMUM_BATCH = 64, +}; + +struct uds_request_queue { + /* Wait queue for synchronizing producers and consumer */ + struct wait_queue_head wait_head; + /* Function to process a request */ + uds_request_queue_processor_fn processor; + /* Queue of new incoming requests */ + struct funnel_queue *main_queue; + /* Queue of old requests to retry */ + struct funnel_queue *retry_queue; + /* The thread id of the worker thread */ + struct thread *thread; + /* True if the worker was started */ + bool started; + /* When true, requests can be enqueued */ + bool running; + /* A flag set when the worker is waiting without a timeout */ + atomic_t dormant; +}; + +static inline struct uds_request *poll_queues(struct uds_request_queue *queue) +{ + struct funnel_queue_entry *entry; + + entry = vdo_funnel_queue_poll(queue->retry_queue); + if (entry != NULL) + return container_of(entry, struct uds_request, queue_link); + + entry = vdo_funnel_queue_poll(queue->main_queue); + if (entry != NULL) + return container_of(entry, struct uds_request, queue_link); + + return NULL; +} + +static inline bool are_queues_idle(struct uds_request_queue *queue) +{ + return vdo_is_funnel_queue_idle(queue->retry_queue) && + vdo_is_funnel_queue_idle(queue->main_queue); +} + +/* + * Determine if there is a next request to process, and return it if there is. Also return flags + * indicating whether the worker thread can sleep (for the use of wait_event() macros) and whether + * the thread did sleep before returning a new request. + */ +static inline bool dequeue_request(struct uds_request_queue *queue, + struct uds_request **request_ptr, bool *waited_ptr) +{ + struct uds_request *request = poll_queues(queue); + + if (request != NULL) { + *request_ptr = request; + return true; + } + + if (!READ_ONCE(queue->running)) { + /* Wake the worker thread so it can exit. */ + *request_ptr = NULL; + return true; + } + + *request_ptr = NULL; + *waited_ptr = true; + return false; +} + +static void wait_for_request(struct uds_request_queue *queue, bool dormant, + unsigned long timeout, struct uds_request **request, + bool *waited) +{ + if (dormant) { + wait_event_interruptible(queue->wait_head, + (dequeue_request(queue, request, waited) || + !are_queues_idle(queue))); + return; + } + + wait_event_interruptible_hrtimeout(queue->wait_head, + dequeue_request(queue, request, waited), + ns_to_ktime(timeout)); +} + +static void request_queue_worker(void *arg) +{ + struct uds_request_queue *queue = arg; + struct uds_request *request = NULL; + unsigned long time_batch = DEFAULT_WAIT_TIME; + bool dormant = atomic_read(&queue->dormant); + bool waited = false; + long current_batch = 0; + + for (;;) { + wait_for_request(queue, dormant, time_batch, &request, &waited); + if (likely(request != NULL)) { + current_batch++; + queue->processor(request); + } else if (!READ_ONCE(queue->running)) { + break; + } + + if (dormant) { + /* + * The queue has been roused from dormancy. Clear the flag so enqueuers can + * stop broadcasting. No fence is needed for this transition. + */ + atomic_set(&queue->dormant, false); + dormant = false; + time_batch = DEFAULT_WAIT_TIME; + } else if (waited) { + /* + * We waited for this request to show up. Adjust the wait time to smooth + * out the batch size. + */ + if (current_batch < MINIMUM_BATCH) { + /* + * If the last batch of requests was too small, increase the wait + * time. + */ + time_batch += time_batch / 4; + if (time_batch >= MAXIMUM_WAIT_TIME) { + atomic_set(&queue->dormant, true); + dormant = true; + } + } else if (current_batch > MAXIMUM_BATCH) { + /* + * If the last batch of requests was too large, decrease the wait + * time. + */ + time_batch -= time_batch / 4; + if (time_batch < MINIMUM_WAIT_TIME) + time_batch = MINIMUM_WAIT_TIME; + } + current_batch = 0; + } + } + + /* + * Ensure that we process any remaining requests that were enqueued before trying to shut + * down. The corresponding write barrier is in uds_request_queue_finish(). + */ + smp_rmb(); + while ((request = poll_queues(queue)) != NULL) + queue->processor(request); +} + +int uds_make_request_queue(const char *queue_name, + uds_request_queue_processor_fn processor, + struct uds_request_queue **queue_ptr) +{ + int result; + struct uds_request_queue *queue; + + result = vdo_allocate(1, struct uds_request_queue, __func__, &queue); + if (result != VDO_SUCCESS) + return result; + + queue->processor = processor; + queue->running = true; + atomic_set(&queue->dormant, false); + init_waitqueue_head(&queue->wait_head); + + result = vdo_make_funnel_queue(&queue->main_queue); + if (result != VDO_SUCCESS) { + uds_request_queue_finish(queue); + return result; + } + + result = vdo_make_funnel_queue(&queue->retry_queue); + if (result != VDO_SUCCESS) { + uds_request_queue_finish(queue); + return result; + } + + result = vdo_create_thread(request_queue_worker, queue, queue_name, + &queue->thread); + if (result != VDO_SUCCESS) { + uds_request_queue_finish(queue); + return result; + } + + queue->started = true; + *queue_ptr = queue; + return UDS_SUCCESS; +} + +static inline void wake_up_worker(struct uds_request_queue *queue) +{ + if (wq_has_sleeper(&queue->wait_head)) + wake_up(&queue->wait_head); +} + +void uds_request_queue_enqueue(struct uds_request_queue *queue, + struct uds_request *request) +{ + struct funnel_queue *sub_queue; + bool unbatched = request->unbatched; + + sub_queue = request->requeued ? queue->retry_queue : queue->main_queue; + vdo_funnel_queue_put(sub_queue, &request->queue_link); + + /* + * We must wake the worker thread when it is dormant. A read fence isn't needed here since + * we know the queue operation acts as one. + */ + if (atomic_read(&queue->dormant) || unbatched) + wake_up_worker(queue); +} + +void uds_request_queue_finish(struct uds_request_queue *queue) +{ + if (queue == NULL) + return; + + /* + * This memory barrier ensures that any requests we queued will be seen. The point is that + * when dequeue_request() sees the following update to the running flag, it will also be + * able to see any change we made to a next field in the funnel queue entry. The + * corresponding read barrier is in request_queue_worker(). + */ + smp_wmb(); + WRITE_ONCE(queue->running, false); + + if (queue->started) { + wake_up_worker(queue); + vdo_join_threads(queue->thread); + } + + vdo_free_funnel_queue(queue->main_queue); + vdo_free_funnel_queue(queue->retry_queue); + vdo_free(queue); +} diff --git a/drivers/md/dm-vdo/indexer/funnel-requestqueue.h b/drivers/md/dm-vdo/indexer/funnel-requestqueue.h new file mode 100644 index 000000000000..9b0f53939b4d --- /dev/null +++ b/drivers/md/dm-vdo/indexer/funnel-requestqueue.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_REQUEST_QUEUE_H +#define UDS_REQUEST_QUEUE_H + +#include "indexer.h" + +/* + * A simple request queue which will handle new requests in the order in which they are received, + * and will attempt to handle requeued requests before new ones. However, the nature of the + * implementation means that it cannot guarantee this ordering; the prioritization is merely a + * hint. + */ + +struct uds_request_queue; + +typedef void (*uds_request_queue_processor_fn)(struct uds_request *); + +int __must_check uds_make_request_queue(const char *queue_name, + uds_request_queue_processor_fn processor, + struct uds_request_queue **queue_ptr); + +void uds_request_queue_enqueue(struct uds_request_queue *queue, + struct uds_request *request); + +void uds_request_queue_finish(struct uds_request_queue *queue); + +#endif /* UDS_REQUEST_QUEUE_H */ diff --git a/drivers/md/dm-vdo/indexer/geometry.c b/drivers/md/dm-vdo/indexer/geometry.c new file mode 100644 index 000000000000..c0575612e820 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/geometry.c @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "geometry.h" + +#include +#include + +#include "errors.h" +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "delta-index.h" +#include "indexer.h" + +/* + * An index volume is divided into a fixed number of fixed-size chapters, each consisting of a + * fixed number of fixed-size pages. The volume layout is defined by two constants and four + * parameters. The constants are that index records are 32 bytes long (16-byte block name plus + * 16-byte metadata) and that open chapter index hash slots are one byte long. The four parameters + * are the number of bytes in a page, the number of record pages in a chapter, the number of + * chapters in a volume, and the number of chapters that are sparse. From these parameters, we can + * derive the rest of the layout and other index properties. + * + * The index volume is sized by its maximum memory footprint. For a dense index, the persistent + * storage is about 10 times the size of the memory footprint. For a sparse index, the persistent + * storage is about 100 times the size of the memory footprint. + * + * For a small index with a memory footprint less than 1GB, there are three possible memory + * configurations: 0.25GB, 0.5GB and 0.75GB. The default geometry for each is 1024 index records + * per 32 KB page, 1024 chapters per volume, and either 64, 128, or 192 record pages per chapter + * (resulting in 6, 13, or 20 index pages per chapter) depending on the memory configuration. For + * the VDO default of a 0.25 GB index, this yields a deduplication window of 256 GB using about 2.5 + * GB for the persistent storage and 256 MB of RAM. + * + * For a larger index with a memory footprint that is a multiple of 1 GB, the geometry is 1024 + * index records per 32 KB page, 256 record pages per chapter, 26 index pages per chapter, and 1024 + * chapters for every GB of memory footprint. For a 1 GB volume, this yields a deduplication window + * of 1 TB using about 9GB of persistent storage and 1 GB of RAM. + * + * The above numbers hold for volumes which have no sparse chapters. A sparse volume has 10 times + * as many chapters as the corresponding non-sparse volume, which provides 10 times the + * deduplication window while using 10 times as much persistent storage as the equivalent + * non-sparse volume with the same memory footprint. + * + * If the volume has been converted from a non-lvm format to an lvm volume, the number of chapters + * per volume will have been reduced by one by eliminating physical chapter 0, and the virtual + * chapter that formerly mapped to physical chapter 0 may be remapped to another physical chapter. + * This remapping is expressed by storing which virtual chapter was remapped, and which physical + * chapter it was moved to. + */ + +int uds_make_index_geometry(size_t bytes_per_page, u32 record_pages_per_chapter, + u32 chapters_per_volume, u32 sparse_chapters_per_volume, + u64 remapped_virtual, u64 remapped_physical, + struct index_geometry **geometry_ptr) +{ + int result; + struct index_geometry *geometry; + + result = vdo_allocate(1, struct index_geometry, "geometry", &geometry); + if (result != VDO_SUCCESS) + return result; + + geometry->bytes_per_page = bytes_per_page; + geometry->record_pages_per_chapter = record_pages_per_chapter; + geometry->chapters_per_volume = chapters_per_volume; + geometry->sparse_chapters_per_volume = sparse_chapters_per_volume; + geometry->dense_chapters_per_volume = chapters_per_volume - sparse_chapters_per_volume; + geometry->remapped_virtual = remapped_virtual; + geometry->remapped_physical = remapped_physical; + + geometry->records_per_page = bytes_per_page / BYTES_PER_RECORD; + geometry->records_per_chapter = geometry->records_per_page * record_pages_per_chapter; + geometry->records_per_volume = (u64) geometry->records_per_chapter * chapters_per_volume; + + geometry->chapter_mean_delta = 1 << DEFAULT_CHAPTER_MEAN_DELTA_BITS; + geometry->chapter_payload_bits = bits_per(record_pages_per_chapter - 1); + /* + * We want 1 delta list for every 64 records in the chapter. + * The "| 077" ensures that the chapter_delta_list_bits computation + * does not underflow. + */ + geometry->chapter_delta_list_bits = + bits_per((geometry->records_per_chapter - 1) | 077) - 6; + geometry->delta_lists_per_chapter = 1 << geometry->chapter_delta_list_bits; + /* We need enough address bits to achieve the desired mean delta. */ + geometry->chapter_address_bits = + (DEFAULT_CHAPTER_MEAN_DELTA_BITS - + geometry->chapter_delta_list_bits + + bits_per(geometry->records_per_chapter - 1)); + geometry->index_pages_per_chapter = + uds_get_delta_index_page_count(geometry->records_per_chapter, + geometry->delta_lists_per_chapter, + geometry->chapter_mean_delta, + geometry->chapter_payload_bits, + bytes_per_page); + + geometry->pages_per_chapter = geometry->index_pages_per_chapter + record_pages_per_chapter; + geometry->pages_per_volume = geometry->pages_per_chapter * chapters_per_volume; + geometry->bytes_per_volume = + bytes_per_page * (geometry->pages_per_volume + HEADER_PAGES_PER_VOLUME); + + *geometry_ptr = geometry; + return UDS_SUCCESS; +} + +int uds_copy_index_geometry(struct index_geometry *source, + struct index_geometry **geometry_ptr) +{ + return uds_make_index_geometry(source->bytes_per_page, + source->record_pages_per_chapter, + source->chapters_per_volume, + source->sparse_chapters_per_volume, + source->remapped_virtual, source->remapped_physical, + geometry_ptr); +} + +void uds_free_index_geometry(struct index_geometry *geometry) +{ + vdo_free(geometry); +} + +u32 __must_check uds_map_to_physical_chapter(const struct index_geometry *geometry, + u64 virtual_chapter) +{ + u64 delta; + + if (!uds_is_reduced_index_geometry(geometry)) + return virtual_chapter % geometry->chapters_per_volume; + + if (likely(virtual_chapter > geometry->remapped_virtual)) { + delta = virtual_chapter - geometry->remapped_virtual; + if (likely(delta > geometry->remapped_physical)) + return delta % geometry->chapters_per_volume; + else + return delta - 1; + } + + if (virtual_chapter == geometry->remapped_virtual) + return geometry->remapped_physical; + + delta = geometry->remapped_virtual - virtual_chapter; + if (delta < geometry->chapters_per_volume) + return geometry->chapters_per_volume - delta; + + /* This chapter is so old the answer doesn't matter. */ + return 0; +} + +/* Check whether any sparse chapters are in use. */ +bool uds_has_sparse_chapters(const struct index_geometry *geometry, + u64 oldest_virtual_chapter, u64 newest_virtual_chapter) +{ + return uds_is_sparse_index_geometry(geometry) && + ((newest_virtual_chapter - oldest_virtual_chapter + 1) > + geometry->dense_chapters_per_volume); +} + +bool uds_is_chapter_sparse(const struct index_geometry *geometry, + u64 oldest_virtual_chapter, u64 newest_virtual_chapter, + u64 virtual_chapter_number) +{ + return uds_has_sparse_chapters(geometry, oldest_virtual_chapter, + newest_virtual_chapter) && + ((virtual_chapter_number + geometry->dense_chapters_per_volume) <= + newest_virtual_chapter); +} + +/* Calculate how many chapters to expire after opening the newest chapter. */ +u32 uds_chapters_to_expire(const struct index_geometry *geometry, u64 newest_chapter) +{ + /* If the index isn't full yet, don't expire anything. */ + if (newest_chapter < geometry->chapters_per_volume) + return 0; + + /* If a chapter is out of order... */ + if (geometry->remapped_physical > 0) { + u64 oldest_chapter = newest_chapter - geometry->chapters_per_volume; + + /* + * ... expire an extra chapter when expiring the moved chapter to free physical + * space for the new chapter ... + */ + if (oldest_chapter == geometry->remapped_virtual) + return 2; + + /* + * ... but don't expire anything when the new chapter will use the physical chapter + * freed by expiring the moved chapter. + */ + if (oldest_chapter == (geometry->remapped_virtual + geometry->remapped_physical)) + return 0; + } + + /* Normally, just expire one. */ + return 1; +} diff --git a/drivers/md/dm-vdo/indexer/geometry.h b/drivers/md/dm-vdo/indexer/geometry.h new file mode 100644 index 000000000000..a2ecdb238cf2 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/geometry.h @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_INDEX_GEOMETRY_H +#define UDS_INDEX_GEOMETRY_H + +#include "indexer.h" + +/* + * The index_geometry records parameters that define the layout of a UDS index volume, and the size and + * shape of various index structures. It is created when the index is created, and is referenced by + * many index sub-components. + */ + +struct index_geometry { + /* Size of a chapter page, in bytes */ + size_t bytes_per_page; + /* Number of record pages in a chapter */ + u32 record_pages_per_chapter; + /* Total number of chapters in a volume */ + u32 chapters_per_volume; + /* Number of sparsely-indexed chapters in a volume */ + u32 sparse_chapters_per_volume; + /* Number of bits used to determine delta list numbers */ + u8 chapter_delta_list_bits; + /* Virtual chapter remapped from physical chapter 0 */ + u64 remapped_virtual; + /* New physical chapter where the remapped chapter can be found */ + u64 remapped_physical; + + /* + * The following properties are derived from the ones above, but they are computed and + * recorded as fields for convenience. + */ + /* Total number of pages in a volume, excluding the header */ + u32 pages_per_volume; + /* Total number of bytes in a volume, including the header */ + size_t bytes_per_volume; + /* Number of pages in a chapter */ + u32 pages_per_chapter; + /* Number of index pages in a chapter index */ + u32 index_pages_per_chapter; + /* Number of records that fit on a page */ + u32 records_per_page; + /* Number of records that fit in a chapter */ + u32 records_per_chapter; + /* Number of records that fit in a volume */ + u64 records_per_volume; + /* Number of delta lists per chapter index */ + u32 delta_lists_per_chapter; + /* Mean delta for chapter indexes */ + u32 chapter_mean_delta; + /* Number of bits needed for record page numbers */ + u8 chapter_payload_bits; + /* Number of bits used to compute addresses for chapter delta lists */ + u8 chapter_address_bits; + /* Number of densely-indexed chapters in a volume */ + u32 dense_chapters_per_volume; +}; + +enum { + /* The number of bytes in a record (name + metadata) */ + BYTES_PER_RECORD = (UDS_RECORD_NAME_SIZE + UDS_RECORD_DATA_SIZE), + + /* The default length of a page in a chapter, in bytes */ + DEFAULT_BYTES_PER_PAGE = 1024 * BYTES_PER_RECORD, + + /* The default maximum number of records per page */ + DEFAULT_RECORDS_PER_PAGE = DEFAULT_BYTES_PER_PAGE / BYTES_PER_RECORD, + + /* The default number of record pages in a chapter */ + DEFAULT_RECORD_PAGES_PER_CHAPTER = 256, + + /* The default number of record pages in a chapter for a small index */ + SMALL_RECORD_PAGES_PER_CHAPTER = 64, + + /* The default number of chapters in a volume */ + DEFAULT_CHAPTERS_PER_VOLUME = 1024, + + /* The default number of sparsely-indexed chapters in a volume */ + DEFAULT_SPARSE_CHAPTERS_PER_VOLUME = 0, + + /* The log2 of the default mean delta */ + DEFAULT_CHAPTER_MEAN_DELTA_BITS = 16, + + /* The log2 of the number of delta lists in a large chapter */ + DEFAULT_CHAPTER_DELTA_LIST_BITS = 12, + + /* The log2 of the number of delta lists in a small chapter */ + SMALL_CHAPTER_DELTA_LIST_BITS = 10, + + /* The number of header pages per volume */ + HEADER_PAGES_PER_VOLUME = 1, +}; + +int __must_check uds_make_index_geometry(size_t bytes_per_page, u32 record_pages_per_chapter, + u32 chapters_per_volume, + u32 sparse_chapters_per_volume, u64 remapped_virtual, + u64 remapped_physical, + struct index_geometry **geometry_ptr); + +int __must_check uds_copy_index_geometry(struct index_geometry *source, + struct index_geometry **geometry_ptr); + +void uds_free_index_geometry(struct index_geometry *geometry); + +u32 __must_check uds_map_to_physical_chapter(const struct index_geometry *geometry, + u64 virtual_chapter); + +/* + * Check whether this geometry is reduced by a chapter. This will only be true if the volume was + * converted from a non-lvm volume to an lvm volume. + */ +static inline bool __must_check +uds_is_reduced_index_geometry(const struct index_geometry *geometry) +{ + return !!(geometry->chapters_per_volume & 1); +} + +static inline bool __must_check +uds_is_sparse_index_geometry(const struct index_geometry *geometry) +{ + return geometry->sparse_chapters_per_volume > 0; +} + +bool __must_check uds_has_sparse_chapters(const struct index_geometry *geometry, + u64 oldest_virtual_chapter, + u64 newest_virtual_chapter); + +bool __must_check uds_is_chapter_sparse(const struct index_geometry *geometry, + u64 oldest_virtual_chapter, + u64 newest_virtual_chapter, + u64 virtual_chapter_number); + +u32 __must_check uds_chapters_to_expire(const struct index_geometry *geometry, + u64 newest_chapter); + +#endif /* UDS_INDEX_GEOMETRY_H */ diff --git a/drivers/md/dm-vdo/indexer/hash-utils.h b/drivers/md/dm-vdo/indexer/hash-utils.h new file mode 100644 index 000000000000..6a8dd8ffea6c --- /dev/null +++ b/drivers/md/dm-vdo/indexer/hash-utils.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_HASH_UTILS_H +#define UDS_HASH_UTILS_H + +#include "numeric.h" + +#include "geometry.h" +#include "indexer.h" + +/* Utilities for extracting portions of a request name for various uses. */ + +/* How various portions of a record name are apportioned. */ +enum { + VOLUME_INDEX_BYTES_OFFSET = 0, + VOLUME_INDEX_BYTES_COUNT = 8, + CHAPTER_INDEX_BYTES_OFFSET = 8, + CHAPTER_INDEX_BYTES_COUNT = 6, + SAMPLE_BYTES_OFFSET = 14, + SAMPLE_BYTES_COUNT = 2, +}; + +static inline u64 uds_extract_chapter_index_bytes(const struct uds_record_name *name) +{ + const u8 *chapter_bits = &name->name[CHAPTER_INDEX_BYTES_OFFSET]; + u64 bytes = (u64) get_unaligned_be16(chapter_bits) << 32; + + bytes |= get_unaligned_be32(chapter_bits + 2); + return bytes; +} + +static inline u64 uds_extract_volume_index_bytes(const struct uds_record_name *name) +{ + return get_unaligned_be64(&name->name[VOLUME_INDEX_BYTES_OFFSET]); +} + +static inline u32 uds_extract_sampling_bytes(const struct uds_record_name *name) +{ + return get_unaligned_be16(&name->name[SAMPLE_BYTES_OFFSET]); +} + +/* Compute the chapter delta list for a given name. */ +static inline u32 uds_hash_to_chapter_delta_list(const struct uds_record_name *name, + const struct index_geometry *geometry) +{ + return ((uds_extract_chapter_index_bytes(name) >> geometry->chapter_address_bits) & + ((1 << geometry->chapter_delta_list_bits) - 1)); +} + +/* Compute the chapter delta address for a given name. */ +static inline u32 uds_hash_to_chapter_delta_address(const struct uds_record_name *name, + const struct index_geometry *geometry) +{ + return uds_extract_chapter_index_bytes(name) & ((1 << geometry->chapter_address_bits) - 1); +} + +static inline unsigned int uds_name_to_hash_slot(const struct uds_record_name *name, + unsigned int slot_count) +{ + return (unsigned int) (uds_extract_chapter_index_bytes(name) % slot_count); +} + +#endif /* UDS_HASH_UTILS_H */ diff --git a/drivers/md/dm-vdo/indexer/index-layout.c b/drivers/md/dm-vdo/indexer/index-layout.c new file mode 100644 index 000000000000..627adc24af3b --- /dev/null +++ b/drivers/md/dm-vdo/indexer/index-layout.c @@ -0,0 +1,1765 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "index-layout.h" + +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "murmurhash3.h" +#include "numeric.h" +#include "time-utils.h" + +#include "config.h" +#include "open-chapter.h" +#include "volume-index.h" + +/* + * The UDS layout on storage media is divided into a number of fixed-size regions, the sizes of + * which are computed when the index is created. Every header and region begins on 4K block + * boundary. Save regions are further sub-divided into regions of their own. + * + * Each region has a kind and an instance number. Some kinds only have one instance and therefore + * use RL_SOLE_INSTANCE (-1) as the instance number. The RL_KIND_INDEX used to use instances to + * represent sub-indices; now, however there is only ever one sub-index and therefore one instance. + * The RL_KIND_VOLUME_INDEX uses instances to record which zone is being saved. + * + * Every region header has a type and version. + * + * +-+-+---------+--------+--------+-+ + * | | | I N D E X 0 101, 0 | | + * |H|C+---------+--------+--------+S| + * |D|f| Volume | Save | Save |e| + * |R|g| Region | Region | Region |a| + * | | | 201, -1 | 202, 0 | 202, 1 |l| + * +-+-+--------+---------+--------+-+ + * + * The header contains the encoded region layout table as well as some index configuration data. + * The sub-index region and its subdivisions are maintained in the same table. + * + * There are two save regions to preserve the old state in case saving the new state is incomplete. + * They are used in alternation. Each save region is further divided into sub-regions. + * + * +-+-----+------+------+-----+-----+ + * |H| IPM | MI | MI | | OC | + * |D| | zone | zone | ... | | + * |R| 301 | 302 | 302 | | 303 | + * | | -1 | 0 | 1 | | -1 | + * +-+-----+------+------+-----+-----+ + * + * The header contains the encoded region layout table as well as index state data for that save. + * Each save also has a unique nonce. + */ + +#define MAGIC_SIZE 32 +#define NONCE_INFO_SIZE 32 +#define MAX_SAVES 2 + +enum region_kind { + RL_KIND_EMPTY = 0, + RL_KIND_HEADER = 1, + RL_KIND_CONFIG = 100, + RL_KIND_INDEX = 101, + RL_KIND_SEAL = 102, + RL_KIND_VOLUME = 201, + RL_KIND_SAVE = 202, + RL_KIND_INDEX_PAGE_MAP = 301, + RL_KIND_VOLUME_INDEX = 302, + RL_KIND_OPEN_CHAPTER = 303, +}; + +/* Some region types are historical and are no longer used. */ +enum region_type { + RH_TYPE_FREE = 0, /* unused */ + RH_TYPE_SUPER = 1, + RH_TYPE_SAVE = 2, + RH_TYPE_CHECKPOINT = 3, /* unused */ + RH_TYPE_UNSAVED = 4, +}; + +#define RL_SOLE_INSTANCE 65535 + +/* + * Super block version 2 is the first released version. + * + * Super block version 3 is the normal version used from RHEL 8.2 onwards. + * + * Super block versions 4 through 6 were incremental development versions and + * are not supported. + * + * Super block version 7 is used for volumes which have been reduced in size by one chapter in + * order to make room to prepend LVM metadata to a volume originally created without lvm. This + * allows the index to retain most its deduplication records. + */ +#define SUPER_VERSION_MINIMUM 3 +#define SUPER_VERSION_CURRENT 3 +#define SUPER_VERSION_MAXIMUM 7 + +static const u8 LAYOUT_MAGIC[MAGIC_SIZE] = "*ALBIREO*SINGLE*FILE*LAYOUT*001*"; +static const u64 REGION_MAGIC = 0x416c6252676e3031; /* 'AlbRgn01' */ + +struct region_header { + u64 magic; + u64 region_blocks; + u16 type; + /* Currently always version 1 */ + u16 version; + u16 region_count; + u16 payload; +}; + +struct layout_region { + u64 start_block; + u64 block_count; + u32 __unused; + u16 kind; + u16 instance; +}; + +struct region_table { + size_t encoded_size; + struct region_header header; + struct layout_region regions[]; +}; + +struct index_save_data { + u64 timestamp; + u64 nonce; + /* Currently always version 1 */ + u32 version; + u32 unused__; +}; + +struct index_state_version { + s32 signature; + s32 version_id; +}; + +static const struct index_state_version INDEX_STATE_VERSION_301 = { + .signature = -1, + .version_id = 301, +}; + +struct index_state_data301 { + struct index_state_version version; + u64 newest_chapter; + u64 oldest_chapter; + u64 last_save; + u32 unused; + u32 padding; +}; + +struct index_save_layout { + unsigned int zone_count; + struct layout_region index_save; + struct layout_region header; + struct layout_region index_page_map; + struct layout_region free_space; + struct layout_region volume_index_zones[MAX_ZONES]; + struct layout_region open_chapter; + struct index_save_data save_data; + struct index_state_data301 state_data; +}; + +struct sub_index_layout { + u64 nonce; + struct layout_region sub_index; + struct layout_region volume; + struct index_save_layout *saves; +}; + +struct super_block_data { + u8 magic_label[MAGIC_SIZE]; + u8 nonce_info[NONCE_INFO_SIZE]; + u64 nonce; + u32 version; + u32 block_size; + u16 index_count; + u16 max_saves; + /* Padding reflects a blank field on permanent storage */ + u8 padding[4]; + u64 open_chapter_blocks; + u64 page_map_blocks; + u64 volume_offset; + u64 start_offset; +}; + +struct index_layout { + struct io_factory *factory; + size_t factory_size; + off_t offset; + struct super_block_data super; + struct layout_region header; + struct layout_region config; + struct sub_index_layout index; + struct layout_region seal; + u64 total_blocks; +}; + +struct save_layout_sizes { + unsigned int save_count; + size_t block_size; + u64 volume_blocks; + u64 volume_index_blocks; + u64 page_map_blocks; + u64 open_chapter_blocks; + u64 save_blocks; + u64 sub_index_blocks; + u64 total_blocks; + size_t total_size; +}; + +static inline bool is_converted_super_block(struct super_block_data *super) +{ + return super->version == 7; +} + +static int __must_check compute_sizes(const struct uds_configuration *config, + struct save_layout_sizes *sls) +{ + int result; + struct index_geometry *geometry = config->geometry; + + memset(sls, 0, sizeof(*sls)); + sls->save_count = MAX_SAVES; + sls->block_size = UDS_BLOCK_SIZE; + sls->volume_blocks = geometry->bytes_per_volume / sls->block_size; + + result = uds_compute_volume_index_save_blocks(config, sls->block_size, + &sls->volume_index_blocks); + if (result != UDS_SUCCESS) + return vdo_log_error_strerror(result, "cannot compute index save size"); + + sls->page_map_blocks = + DIV_ROUND_UP(uds_compute_index_page_map_save_size(geometry), + sls->block_size); + sls->open_chapter_blocks = + DIV_ROUND_UP(uds_compute_saved_open_chapter_size(geometry), + sls->block_size); + sls->save_blocks = + 1 + (sls->volume_index_blocks + sls->page_map_blocks + sls->open_chapter_blocks); + sls->sub_index_blocks = sls->volume_blocks + (sls->save_count * sls->save_blocks); + sls->total_blocks = 3 + sls->sub_index_blocks; + sls->total_size = sls->total_blocks * sls->block_size; + + return UDS_SUCCESS; +} + +int uds_compute_index_size(const struct uds_parameters *parameters, u64 *index_size) +{ + int result; + struct uds_configuration *index_config; + struct save_layout_sizes sizes; + + if (index_size == NULL) { + vdo_log_error("Missing output size pointer"); + return -EINVAL; + } + + result = uds_make_configuration(parameters, &index_config); + if (result != UDS_SUCCESS) { + vdo_log_error_strerror(result, "cannot compute index size"); + return uds_status_to_errno(result); + } + + result = compute_sizes(index_config, &sizes); + uds_free_configuration(index_config); + if (result != UDS_SUCCESS) + return uds_status_to_errno(result); + + *index_size = sizes.total_size; + return UDS_SUCCESS; +} + +/* Create unique data using the current time and a pseudorandom number. */ +static void create_unique_nonce_data(u8 *buffer) +{ + ktime_t now = current_time_ns(CLOCK_REALTIME); + u32 rand; + size_t offset = 0; + + get_random_bytes(&rand, sizeof(u32)); + memcpy(buffer + offset, &now, sizeof(now)); + offset += sizeof(now); + memcpy(buffer + offset, &rand, sizeof(rand)); + offset += sizeof(rand); + while (offset < NONCE_INFO_SIZE) { + size_t len = min(NONCE_INFO_SIZE - offset, offset); + + memcpy(buffer + offset, buffer, len); + offset += len; + } +} + +static u64 hash_stuff(u64 start, const void *data, size_t len) +{ + u32 seed = start ^ (start >> 27); + u8 hash_buffer[16]; + + murmurhash3_128(data, len, seed, hash_buffer); + return get_unaligned_le64(hash_buffer + 4); +} + +/* Generate a primary nonce from the provided data. */ +static u64 generate_primary_nonce(const void *data, size_t len) +{ + return hash_stuff(0xa1b1e0fc, data, len); +} + +/* + * Deterministically generate a secondary nonce from an existing nonce and some arbitrary data by + * hashing the original nonce and the data to produce a new nonce. + */ +static u64 generate_secondary_nonce(u64 nonce, const void *data, size_t len) +{ + return hash_stuff(nonce + 1, data, len); +} + +static int __must_check open_layout_reader(struct index_layout *layout, + struct layout_region *lr, off_t offset, + struct buffered_reader **reader_ptr) +{ + return uds_make_buffered_reader(layout->factory, lr->start_block + offset, + lr->block_count, reader_ptr); +} + +static int open_region_reader(struct index_layout *layout, struct layout_region *region, + struct buffered_reader **reader_ptr) +{ + return open_layout_reader(layout, region, -layout->super.start_offset, + reader_ptr); +} + +static int __must_check open_layout_writer(struct index_layout *layout, + struct layout_region *lr, off_t offset, + struct buffered_writer **writer_ptr) +{ + return uds_make_buffered_writer(layout->factory, lr->start_block + offset, + lr->block_count, writer_ptr); +} + +static int open_region_writer(struct index_layout *layout, struct layout_region *region, + struct buffered_writer **writer_ptr) +{ + return open_layout_writer(layout, region, -layout->super.start_offset, + writer_ptr); +} + +static void generate_super_block_data(struct save_layout_sizes *sls, + struct super_block_data *super) +{ + memset(super, 0, sizeof(*super)); + memcpy(super->magic_label, LAYOUT_MAGIC, MAGIC_SIZE); + create_unique_nonce_data(super->nonce_info); + + super->nonce = generate_primary_nonce(super->nonce_info, + sizeof(super->nonce_info)); + super->version = SUPER_VERSION_CURRENT; + super->block_size = sls->block_size; + super->index_count = 1; + super->max_saves = sls->save_count; + super->open_chapter_blocks = sls->open_chapter_blocks; + super->page_map_blocks = sls->page_map_blocks; + super->volume_offset = 0; + super->start_offset = 0; +} + +static void define_sub_index_nonce(struct index_layout *layout) +{ + struct sub_index_nonce_data { + u64 offset; + u16 index_id; + }; + struct sub_index_layout *sil = &layout->index; + u64 primary_nonce = layout->super.nonce; + u8 buffer[sizeof(struct sub_index_nonce_data)] = { 0 }; + size_t offset = 0; + + encode_u64_le(buffer, &offset, sil->sub_index.start_block); + encode_u16_le(buffer, &offset, 0); + sil->nonce = generate_secondary_nonce(primary_nonce, buffer, sizeof(buffer)); + if (sil->nonce == 0) { + sil->nonce = generate_secondary_nonce(~primary_nonce + 1, buffer, + sizeof(buffer)); + } +} + +static void setup_sub_index(struct index_layout *layout, u64 start_block, + struct save_layout_sizes *sls) +{ + struct sub_index_layout *sil = &layout->index; + u64 next_block = start_block; + unsigned int i; + + sil->sub_index = (struct layout_region) { + .start_block = start_block, + .block_count = sls->sub_index_blocks, + .kind = RL_KIND_INDEX, + .instance = 0, + }; + + sil->volume = (struct layout_region) { + .start_block = next_block, + .block_count = sls->volume_blocks, + .kind = RL_KIND_VOLUME, + .instance = RL_SOLE_INSTANCE, + }; + + next_block += sls->volume_blocks; + + for (i = 0; i < sls->save_count; i++) { + sil->saves[i].index_save = (struct layout_region) { + .start_block = next_block, + .block_count = sls->save_blocks, + .kind = RL_KIND_SAVE, + .instance = i, + }; + + next_block += sls->save_blocks; + } + + define_sub_index_nonce(layout); +} + +static void initialize_layout(struct index_layout *layout, struct save_layout_sizes *sls) +{ + u64 next_block = layout->offset / sls->block_size; + + layout->total_blocks = sls->total_blocks; + generate_super_block_data(sls, &layout->super); + layout->header = (struct layout_region) { + .start_block = next_block++, + .block_count = 1, + .kind = RL_KIND_HEADER, + .instance = RL_SOLE_INSTANCE, + }; + + layout->config = (struct layout_region) { + .start_block = next_block++, + .block_count = 1, + .kind = RL_KIND_CONFIG, + .instance = RL_SOLE_INSTANCE, + }; + + setup_sub_index(layout, next_block, sls); + next_block += sls->sub_index_blocks; + + layout->seal = (struct layout_region) { + .start_block = next_block, + .block_count = 1, + .kind = RL_KIND_SEAL, + .instance = RL_SOLE_INSTANCE, + }; +} + +static int __must_check make_index_save_region_table(struct index_save_layout *isl, + struct region_table **table_ptr) +{ + int result; + unsigned int z; + struct region_table *table; + struct layout_region *lr; + u16 region_count; + size_t payload; + size_t type; + + if (isl->zone_count > 0) { + /* + * Normal save regions: header, page map, volume index zones, + * open chapter, and possibly free space. + */ + region_count = 3 + isl->zone_count; + if (isl->free_space.block_count > 0) + region_count++; + + payload = sizeof(isl->save_data) + sizeof(isl->state_data); + type = RH_TYPE_SAVE; + } else { + /* Empty save regions: header, page map, free space. */ + region_count = 3; + payload = sizeof(isl->save_data); + type = RH_TYPE_UNSAVED; + } + + result = vdo_allocate_extended(struct region_table, region_count, + struct layout_region, + "layout region table for ISL", &table); + if (result != VDO_SUCCESS) + return result; + + lr = &table->regions[0]; + *lr++ = isl->header; + *lr++ = isl->index_page_map; + for (z = 0; z < isl->zone_count; z++) + *lr++ = isl->volume_index_zones[z]; + + if (isl->zone_count > 0) + *lr++ = isl->open_chapter; + + if (isl->free_space.block_count > 0) + *lr++ = isl->free_space; + + table->header = (struct region_header) { + .magic = REGION_MAGIC, + .region_blocks = isl->index_save.block_count, + .type = type, + .version = 1, + .region_count = region_count, + .payload = payload, + }; + + table->encoded_size = (sizeof(struct region_header) + payload + + region_count * sizeof(struct layout_region)); + *table_ptr = table; + return UDS_SUCCESS; +} + +static void encode_region_table(u8 *buffer, size_t *offset, struct region_table *table) +{ + unsigned int i; + + encode_u64_le(buffer, offset, REGION_MAGIC); + encode_u64_le(buffer, offset, table->header.region_blocks); + encode_u16_le(buffer, offset, table->header.type); + encode_u16_le(buffer, offset, table->header.version); + encode_u16_le(buffer, offset, table->header.region_count); + encode_u16_le(buffer, offset, table->header.payload); + + for (i = 0; i < table->header.region_count; i++) { + encode_u64_le(buffer, offset, table->regions[i].start_block); + encode_u64_le(buffer, offset, table->regions[i].block_count); + encode_u32_le(buffer, offset, 0); + encode_u16_le(buffer, offset, table->regions[i].kind); + encode_u16_le(buffer, offset, table->regions[i].instance); + } +} + +static int __must_check write_index_save_header(struct index_save_layout *isl, + struct region_table *table, + struct buffered_writer *writer) +{ + int result; + u8 *buffer; + size_t offset = 0; + + result = vdo_allocate(table->encoded_size, u8, "index save data", &buffer); + if (result != VDO_SUCCESS) + return result; + + encode_region_table(buffer, &offset, table); + encode_u64_le(buffer, &offset, isl->save_data.timestamp); + encode_u64_le(buffer, &offset, isl->save_data.nonce); + encode_u32_le(buffer, &offset, isl->save_data.version); + encode_u32_le(buffer, &offset, 0); + if (isl->zone_count > 0) { + encode_u32_le(buffer, &offset, INDEX_STATE_VERSION_301.signature); + encode_u32_le(buffer, &offset, INDEX_STATE_VERSION_301.version_id); + encode_u64_le(buffer, &offset, isl->state_data.newest_chapter); + encode_u64_le(buffer, &offset, isl->state_data.oldest_chapter); + encode_u64_le(buffer, &offset, isl->state_data.last_save); + encode_u64_le(buffer, &offset, 0); + } + + result = uds_write_to_buffered_writer(writer, buffer, offset); + vdo_free(buffer); + if (result != UDS_SUCCESS) + return result; + + return uds_flush_buffered_writer(writer); +} + +static int write_index_save_layout(struct index_layout *layout, + struct index_save_layout *isl) +{ + int result; + struct region_table *table; + struct buffered_writer *writer; + + result = make_index_save_region_table(isl, &table); + if (result != UDS_SUCCESS) + return result; + + result = open_region_writer(layout, &isl->header, &writer); + if (result != UDS_SUCCESS) { + vdo_free(table); + return result; + } + + result = write_index_save_header(isl, table, writer); + vdo_free(table); + uds_free_buffered_writer(writer); + + return result; +} + +static void reset_index_save_layout(struct index_save_layout *isl, u64 page_map_blocks) +{ + u64 free_blocks; + u64 next_block = isl->index_save.start_block; + + isl->zone_count = 0; + memset(&isl->save_data, 0, sizeof(isl->save_data)); + + isl->header = (struct layout_region) { + .start_block = next_block++, + .block_count = 1, + .kind = RL_KIND_HEADER, + .instance = RL_SOLE_INSTANCE, + }; + + isl->index_page_map = (struct layout_region) { + .start_block = next_block, + .block_count = page_map_blocks, + .kind = RL_KIND_INDEX_PAGE_MAP, + .instance = RL_SOLE_INSTANCE, + }; + + next_block += page_map_blocks; + + free_blocks = isl->index_save.block_count - page_map_blocks - 1; + isl->free_space = (struct layout_region) { + .start_block = next_block, + .block_count = free_blocks, + .kind = RL_KIND_EMPTY, + .instance = RL_SOLE_INSTANCE, + }; +} + +static int __must_check invalidate_old_save(struct index_layout *layout, + struct index_save_layout *isl) +{ + reset_index_save_layout(isl, layout->super.page_map_blocks); + return write_index_save_layout(layout, isl); +} + +static int discard_index_state_data(struct index_layout *layout) +{ + int result; + int saved_result = UDS_SUCCESS; + unsigned int i; + + for (i = 0; i < layout->super.max_saves; i++) { + result = invalidate_old_save(layout, &layout->index.saves[i]); + if (result != UDS_SUCCESS) + saved_result = result; + } + + if (saved_result != UDS_SUCCESS) { + return vdo_log_error_strerror(result, + "%s: cannot destroy all index saves", + __func__); + } + + return UDS_SUCCESS; +} + +static int __must_check make_layout_region_table(struct index_layout *layout, + struct region_table **table_ptr) +{ + int result; + unsigned int i; + /* Regions: header, config, index, volume, saves, seal */ + u16 region_count = 5 + layout->super.max_saves; + u16 payload; + struct region_table *table; + struct layout_region *lr; + + result = vdo_allocate_extended(struct region_table, region_count, + struct layout_region, "layout region table", + &table); + if (result != VDO_SUCCESS) + return result; + + lr = &table->regions[0]; + *lr++ = layout->header; + *lr++ = layout->config; + *lr++ = layout->index.sub_index; + *lr++ = layout->index.volume; + + for (i = 0; i < layout->super.max_saves; i++) + *lr++ = layout->index.saves[i].index_save; + + *lr++ = layout->seal; + + if (is_converted_super_block(&layout->super)) { + payload = sizeof(struct super_block_data); + } else { + payload = (sizeof(struct super_block_data) - + sizeof(layout->super.volume_offset) - + sizeof(layout->super.start_offset)); + } + + table->header = (struct region_header) { + .magic = REGION_MAGIC, + .region_blocks = layout->total_blocks, + .type = RH_TYPE_SUPER, + .version = 1, + .region_count = region_count, + .payload = payload, + }; + + table->encoded_size = (sizeof(struct region_header) + payload + + region_count * sizeof(struct layout_region)); + *table_ptr = table; + return UDS_SUCCESS; +} + +static int __must_check write_layout_header(struct index_layout *layout, + struct region_table *table, + struct buffered_writer *writer) +{ + int result; + u8 *buffer; + size_t offset = 0; + + result = vdo_allocate(table->encoded_size, u8, "layout data", &buffer); + if (result != VDO_SUCCESS) + return result; + + encode_region_table(buffer, &offset, table); + memcpy(buffer + offset, &layout->super.magic_label, MAGIC_SIZE); + offset += MAGIC_SIZE; + memcpy(buffer + offset, &layout->super.nonce_info, NONCE_INFO_SIZE); + offset += NONCE_INFO_SIZE; + encode_u64_le(buffer, &offset, layout->super.nonce); + encode_u32_le(buffer, &offset, layout->super.version); + encode_u32_le(buffer, &offset, layout->super.block_size); + encode_u16_le(buffer, &offset, layout->super.index_count); + encode_u16_le(buffer, &offset, layout->super.max_saves); + encode_u32_le(buffer, &offset, 0); + encode_u64_le(buffer, &offset, layout->super.open_chapter_blocks); + encode_u64_le(buffer, &offset, layout->super.page_map_blocks); + + if (is_converted_super_block(&layout->super)) { + encode_u64_le(buffer, &offset, layout->super.volume_offset); + encode_u64_le(buffer, &offset, layout->super.start_offset); + } + + result = uds_write_to_buffered_writer(writer, buffer, offset); + vdo_free(buffer); + if (result != UDS_SUCCESS) + return result; + + return uds_flush_buffered_writer(writer); +} + +static int __must_check write_uds_index_config(struct index_layout *layout, + struct uds_configuration *config, + off_t offset) +{ + int result; + struct buffered_writer *writer = NULL; + + result = open_layout_writer(layout, &layout->config, offset, &writer); + if (result != UDS_SUCCESS) + return vdo_log_error_strerror(result, "failed to open config region"); + + result = uds_write_config_contents(writer, config, layout->super.version); + if (result != UDS_SUCCESS) { + uds_free_buffered_writer(writer); + return vdo_log_error_strerror(result, "failed to write config region"); + } + + result = uds_flush_buffered_writer(writer); + if (result != UDS_SUCCESS) { + uds_free_buffered_writer(writer); + return vdo_log_error_strerror(result, "cannot flush config writer"); + } + + uds_free_buffered_writer(writer); + return UDS_SUCCESS; +} + +static int __must_check save_layout(struct index_layout *layout, off_t offset) +{ + int result; + struct buffered_writer *writer = NULL; + struct region_table *table; + + result = make_layout_region_table(layout, &table); + if (result != UDS_SUCCESS) + return result; + + result = open_layout_writer(layout, &layout->header, offset, &writer); + if (result != UDS_SUCCESS) { + vdo_free(table); + return result; + } + + result = write_layout_header(layout, table, writer); + vdo_free(table); + uds_free_buffered_writer(writer); + + return result; +} + +static int create_index_layout(struct index_layout *layout, struct uds_configuration *config) +{ + int result; + struct save_layout_sizes sizes; + + result = compute_sizes(config, &sizes); + if (result != UDS_SUCCESS) + return result; + + result = vdo_allocate(sizes.save_count, struct index_save_layout, __func__, + &layout->index.saves); + if (result != VDO_SUCCESS) + return result; + + initialize_layout(layout, &sizes); + + result = discard_index_state_data(layout); + if (result != UDS_SUCCESS) + return result; + + result = write_uds_index_config(layout, config, 0); + if (result != UDS_SUCCESS) + return result; + + return save_layout(layout, 0); +} + +static u64 generate_index_save_nonce(u64 volume_nonce, struct index_save_layout *isl) +{ + struct save_nonce_data { + struct index_save_data data; + u64 offset; + } nonce_data; + u8 buffer[sizeof(nonce_data)]; + size_t offset = 0; + + encode_u64_le(buffer, &offset, isl->save_data.timestamp); + encode_u64_le(buffer, &offset, 0); + encode_u32_le(buffer, &offset, isl->save_data.version); + encode_u32_le(buffer, &offset, 0U); + encode_u64_le(buffer, &offset, isl->index_save.start_block); + VDO_ASSERT_LOG_ONLY(offset == sizeof(nonce_data), + "%zu bytes encoded of %zu expected", + offset, sizeof(nonce_data)); + return generate_secondary_nonce(volume_nonce, buffer, sizeof(buffer)); +} + +static u64 validate_index_save_layout(struct index_save_layout *isl, u64 volume_nonce) +{ + if ((isl->zone_count == 0) || (isl->save_data.timestamp == 0)) + return 0; + + if (isl->save_data.nonce != generate_index_save_nonce(volume_nonce, isl)) + return 0; + + return isl->save_data.timestamp; +} + +static int find_latest_uds_index_save_slot(struct index_layout *layout, + struct index_save_layout **isl_ptr) +{ + struct index_save_layout *latest = NULL; + struct index_save_layout *isl; + unsigned int i; + u64 save_time = 0; + u64 latest_time = 0; + + for (i = 0; i < layout->super.max_saves; i++) { + isl = &layout->index.saves[i]; + save_time = validate_index_save_layout(isl, layout->index.nonce); + if (save_time > latest_time) { + latest = isl; + latest_time = save_time; + } + } + + if (latest == NULL) { + vdo_log_error("No valid index save found"); + return UDS_INDEX_NOT_SAVED_CLEANLY; + } + + *isl_ptr = latest; + return UDS_SUCCESS; +} + +int uds_discard_open_chapter(struct index_layout *layout) +{ + int result; + struct index_save_layout *isl; + struct buffered_writer *writer; + + result = find_latest_uds_index_save_slot(layout, &isl); + if (result != UDS_SUCCESS) + return result; + + result = open_region_writer(layout, &isl->open_chapter, &writer); + if (result != UDS_SUCCESS) + return result; + + result = uds_write_to_buffered_writer(writer, NULL, UDS_BLOCK_SIZE); + if (result != UDS_SUCCESS) { + uds_free_buffered_writer(writer); + return result; + } + + result = uds_flush_buffered_writer(writer); + uds_free_buffered_writer(writer); + return result; +} + +int uds_load_index_state(struct index_layout *layout, struct uds_index *index) +{ + int result; + unsigned int zone; + struct index_save_layout *isl; + struct buffered_reader *readers[MAX_ZONES]; + + result = find_latest_uds_index_save_slot(layout, &isl); + if (result != UDS_SUCCESS) + return result; + + index->newest_virtual_chapter = isl->state_data.newest_chapter; + index->oldest_virtual_chapter = isl->state_data.oldest_chapter; + index->last_save = isl->state_data.last_save; + + result = open_region_reader(layout, &isl->open_chapter, &readers[0]); + if (result != UDS_SUCCESS) + return result; + + result = uds_load_open_chapter(index, readers[0]); + uds_free_buffered_reader(readers[0]); + if (result != UDS_SUCCESS) + return result; + + for (zone = 0; zone < isl->zone_count; zone++) { + result = open_region_reader(layout, &isl->volume_index_zones[zone], + &readers[zone]); + if (result != UDS_SUCCESS) { + for (; zone > 0; zone--) + uds_free_buffered_reader(readers[zone - 1]); + + return result; + } + } + + result = uds_load_volume_index(index->volume_index, readers, isl->zone_count); + for (zone = 0; zone < isl->zone_count; zone++) + uds_free_buffered_reader(readers[zone]); + if (result != UDS_SUCCESS) + return result; + + result = open_region_reader(layout, &isl->index_page_map, &readers[0]); + if (result != UDS_SUCCESS) + return result; + + result = uds_read_index_page_map(index->volume->index_page_map, readers[0]); + uds_free_buffered_reader(readers[0]); + + return result; +} + +static struct index_save_layout *select_oldest_index_save_layout(struct index_layout *layout) +{ + struct index_save_layout *oldest = NULL; + struct index_save_layout *isl; + unsigned int i; + u64 save_time = 0; + u64 oldest_time = 0; + + for (i = 0; i < layout->super.max_saves; i++) { + isl = &layout->index.saves[i]; + save_time = validate_index_save_layout(isl, layout->index.nonce); + if (oldest == NULL || save_time < oldest_time) { + oldest = isl; + oldest_time = save_time; + } + } + + return oldest; +} + +static void instantiate_index_save_layout(struct index_save_layout *isl, + struct super_block_data *super, + u64 volume_nonce, unsigned int zone_count) +{ + unsigned int z; + u64 next_block; + u64 free_blocks; + u64 volume_index_blocks; + + isl->zone_count = zone_count; + memset(&isl->save_data, 0, sizeof(isl->save_data)); + isl->save_data.timestamp = ktime_to_ms(current_time_ns(CLOCK_REALTIME)); + isl->save_data.version = 1; + isl->save_data.nonce = generate_index_save_nonce(volume_nonce, isl); + + next_block = isl->index_save.start_block; + isl->header = (struct layout_region) { + .start_block = next_block++, + .block_count = 1, + .kind = RL_KIND_HEADER, + .instance = RL_SOLE_INSTANCE, + }; + + isl->index_page_map = (struct layout_region) { + .start_block = next_block, + .block_count = super->page_map_blocks, + .kind = RL_KIND_INDEX_PAGE_MAP, + .instance = RL_SOLE_INSTANCE, + }; + next_block += super->page_map_blocks; + + free_blocks = (isl->index_save.block_count - 1 - + super->page_map_blocks - + super->open_chapter_blocks); + volume_index_blocks = free_blocks / isl->zone_count; + for (z = 0; z < isl->zone_count; z++) { + isl->volume_index_zones[z] = (struct layout_region) { + .start_block = next_block, + .block_count = volume_index_blocks, + .kind = RL_KIND_VOLUME_INDEX, + .instance = z, + }; + + next_block += volume_index_blocks; + free_blocks -= volume_index_blocks; + } + + isl->open_chapter = (struct layout_region) { + .start_block = next_block, + .block_count = super->open_chapter_blocks, + .kind = RL_KIND_OPEN_CHAPTER, + .instance = RL_SOLE_INSTANCE, + }; + + next_block += super->open_chapter_blocks; + + isl->free_space = (struct layout_region) { + .start_block = next_block, + .block_count = free_blocks, + .kind = RL_KIND_EMPTY, + .instance = RL_SOLE_INSTANCE, + }; +} + +static int setup_uds_index_save_slot(struct index_layout *layout, + unsigned int zone_count, + struct index_save_layout **isl_ptr) +{ + int result; + struct index_save_layout *isl; + + isl = select_oldest_index_save_layout(layout); + result = invalidate_old_save(layout, isl); + if (result != UDS_SUCCESS) + return result; + + instantiate_index_save_layout(isl, &layout->super, layout->index.nonce, + zone_count); + + *isl_ptr = isl; + return UDS_SUCCESS; +} + +static void cancel_uds_index_save(struct index_save_layout *isl) +{ + memset(&isl->save_data, 0, sizeof(isl->save_data)); + memset(&isl->state_data, 0, sizeof(isl->state_data)); + isl->zone_count = 0; +} + +int uds_save_index_state(struct index_layout *layout, struct uds_index *index) +{ + int result; + unsigned int zone; + struct index_save_layout *isl; + struct buffered_writer *writers[MAX_ZONES]; + + result = setup_uds_index_save_slot(layout, index->zone_count, &isl); + if (result != UDS_SUCCESS) + return result; + + isl->state_data = (struct index_state_data301) { + .newest_chapter = index->newest_virtual_chapter, + .oldest_chapter = index->oldest_virtual_chapter, + .last_save = index->last_save, + }; + + result = open_region_writer(layout, &isl->open_chapter, &writers[0]); + if (result != UDS_SUCCESS) { + cancel_uds_index_save(isl); + return result; + } + + result = uds_save_open_chapter(index, writers[0]); + uds_free_buffered_writer(writers[0]); + if (result != UDS_SUCCESS) { + cancel_uds_index_save(isl); + return result; + } + + for (zone = 0; zone < index->zone_count; zone++) { + result = open_region_writer(layout, &isl->volume_index_zones[zone], + &writers[zone]); + if (result != UDS_SUCCESS) { + for (; zone > 0; zone--) + uds_free_buffered_writer(writers[zone - 1]); + + cancel_uds_index_save(isl); + return result; + } + } + + result = uds_save_volume_index(index->volume_index, writers, index->zone_count); + for (zone = 0; zone < index->zone_count; zone++) + uds_free_buffered_writer(writers[zone]); + if (result != UDS_SUCCESS) { + cancel_uds_index_save(isl); + return result; + } + + result = open_region_writer(layout, &isl->index_page_map, &writers[0]); + if (result != UDS_SUCCESS) { + cancel_uds_index_save(isl); + return result; + } + + result = uds_write_index_page_map(index->volume->index_page_map, writers[0]); + uds_free_buffered_writer(writers[0]); + if (result != UDS_SUCCESS) { + cancel_uds_index_save(isl); + return result; + } + + return write_index_save_layout(layout, isl); +} + +static int __must_check load_region_table(struct buffered_reader *reader, + struct region_table **table_ptr) +{ + int result; + unsigned int i; + struct region_header header; + struct region_table *table; + u8 buffer[sizeof(struct region_header)]; + size_t offset = 0; + + result = uds_read_from_buffered_reader(reader, buffer, sizeof(buffer)); + if (result != UDS_SUCCESS) + return vdo_log_error_strerror(result, "cannot read region table header"); + + decode_u64_le(buffer, &offset, &header.magic); + decode_u64_le(buffer, &offset, &header.region_blocks); + decode_u16_le(buffer, &offset, &header.type); + decode_u16_le(buffer, &offset, &header.version); + decode_u16_le(buffer, &offset, &header.region_count); + decode_u16_le(buffer, &offset, &header.payload); + + if (header.magic != REGION_MAGIC) + return UDS_NO_INDEX; + + if (header.version != 1) { + return vdo_log_error_strerror(UDS_UNSUPPORTED_VERSION, + "unknown region table version %hu", + header.version); + } + + result = vdo_allocate_extended(struct region_table, header.region_count, + struct layout_region, + "single file layout region table", &table); + if (result != VDO_SUCCESS) + return result; + + table->header = header; + for (i = 0; i < header.region_count; i++) { + u8 region_buffer[sizeof(struct layout_region)]; + + offset = 0; + result = uds_read_from_buffered_reader(reader, region_buffer, + sizeof(region_buffer)); + if (result != UDS_SUCCESS) { + vdo_free(table); + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "cannot read region table layouts"); + } + + decode_u64_le(region_buffer, &offset, &table->regions[i].start_block); + decode_u64_le(region_buffer, &offset, &table->regions[i].block_count); + offset += sizeof(u32); + decode_u16_le(region_buffer, &offset, &table->regions[i].kind); + decode_u16_le(region_buffer, &offset, &table->regions[i].instance); + } + + *table_ptr = table; + return UDS_SUCCESS; +} + +static int __must_check read_super_block_data(struct buffered_reader *reader, + struct index_layout *layout, + size_t saved_size) +{ + int result; + struct super_block_data *super = &layout->super; + u8 *buffer; + size_t offset = 0; + + result = vdo_allocate(saved_size, u8, "super block data", &buffer); + if (result != VDO_SUCCESS) + return result; + + result = uds_read_from_buffered_reader(reader, buffer, saved_size); + if (result != UDS_SUCCESS) { + vdo_free(buffer); + return vdo_log_error_strerror(result, "cannot read region table header"); + } + + memcpy(&super->magic_label, buffer, MAGIC_SIZE); + offset += MAGIC_SIZE; + memcpy(&super->nonce_info, buffer + offset, NONCE_INFO_SIZE); + offset += NONCE_INFO_SIZE; + decode_u64_le(buffer, &offset, &super->nonce); + decode_u32_le(buffer, &offset, &super->version); + decode_u32_le(buffer, &offset, &super->block_size); + decode_u16_le(buffer, &offset, &super->index_count); + decode_u16_le(buffer, &offset, &super->max_saves); + offset += sizeof(u32); + decode_u64_le(buffer, &offset, &super->open_chapter_blocks); + decode_u64_le(buffer, &offset, &super->page_map_blocks); + + if (is_converted_super_block(super)) { + decode_u64_le(buffer, &offset, &super->volume_offset); + decode_u64_le(buffer, &offset, &super->start_offset); + } else { + super->volume_offset = 0; + super->start_offset = 0; + } + + vdo_free(buffer); + + if (memcmp(super->magic_label, LAYOUT_MAGIC, MAGIC_SIZE) != 0) + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "unknown superblock magic label"); + + if ((super->version < SUPER_VERSION_MINIMUM) || + (super->version == 4) || (super->version == 5) || (super->version == 6) || + (super->version > SUPER_VERSION_MAXIMUM)) { + return vdo_log_error_strerror(UDS_UNSUPPORTED_VERSION, + "unknown superblock version number %u", + super->version); + } + + if (super->volume_offset < super->start_offset) { + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "inconsistent offsets (start %llu, volume %llu)", + (unsigned long long) super->start_offset, + (unsigned long long) super->volume_offset); + } + + /* Sub-indexes are no longer used but the layout retains this field. */ + if (super->index_count != 1) { + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "invalid subindex count %u", + super->index_count); + } + + if (generate_primary_nonce(super->nonce_info, sizeof(super->nonce_info)) != super->nonce) { + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "inconsistent superblock nonce"); + } + + return UDS_SUCCESS; +} + +static int __must_check verify_region(struct layout_region *lr, u64 start_block, + enum region_kind kind, unsigned int instance) +{ + if (lr->start_block != start_block) + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "incorrect layout region offset"); + + if (lr->kind != kind) + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "incorrect layout region kind"); + + if (lr->instance != instance) { + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "incorrect layout region instance"); + } + + return UDS_SUCCESS; +} + +static int __must_check verify_sub_index(struct index_layout *layout, u64 start_block, + struct region_table *table) +{ + int result; + unsigned int i; + struct sub_index_layout *sil = &layout->index; + u64 next_block = start_block; + + sil->sub_index = table->regions[2]; + result = verify_region(&sil->sub_index, next_block, RL_KIND_INDEX, 0); + if (result != UDS_SUCCESS) + return result; + + define_sub_index_nonce(layout); + + sil->volume = table->regions[3]; + result = verify_region(&sil->volume, next_block, RL_KIND_VOLUME, + RL_SOLE_INSTANCE); + if (result != UDS_SUCCESS) + return result; + + next_block += sil->volume.block_count + layout->super.volume_offset; + + for (i = 0; i < layout->super.max_saves; i++) { + sil->saves[i].index_save = table->regions[i + 4]; + result = verify_region(&sil->saves[i].index_save, next_block, + RL_KIND_SAVE, i); + if (result != UDS_SUCCESS) + return result; + + next_block += sil->saves[i].index_save.block_count; + } + + next_block -= layout->super.volume_offset; + if (next_block != start_block + sil->sub_index.block_count) { + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "sub index region does not span all saves"); + } + + return UDS_SUCCESS; +} + +static int __must_check reconstitute_layout(struct index_layout *layout, + struct region_table *table, u64 first_block) +{ + int result; + u64 next_block = first_block; + + result = vdo_allocate(layout->super.max_saves, struct index_save_layout, + __func__, &layout->index.saves); + if (result != VDO_SUCCESS) + return result; + + layout->total_blocks = table->header.region_blocks; + + layout->header = table->regions[0]; + result = verify_region(&layout->header, next_block++, RL_KIND_HEADER, + RL_SOLE_INSTANCE); + if (result != UDS_SUCCESS) + return result; + + layout->config = table->regions[1]; + result = verify_region(&layout->config, next_block++, RL_KIND_CONFIG, + RL_SOLE_INSTANCE); + if (result != UDS_SUCCESS) + return result; + + result = verify_sub_index(layout, next_block, table); + if (result != UDS_SUCCESS) + return result; + + next_block += layout->index.sub_index.block_count; + + layout->seal = table->regions[table->header.region_count - 1]; + result = verify_region(&layout->seal, next_block + layout->super.volume_offset, + RL_KIND_SEAL, RL_SOLE_INSTANCE); + if (result != UDS_SUCCESS) + return result; + + if (++next_block != (first_block + layout->total_blocks)) { + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "layout table does not span total blocks"); + } + + return UDS_SUCCESS; +} + +static int __must_check load_super_block(struct index_layout *layout, size_t block_size, + u64 first_block, struct buffered_reader *reader) +{ + int result; + struct region_table *table = NULL; + struct super_block_data *super = &layout->super; + + result = load_region_table(reader, &table); + if (result != UDS_SUCCESS) + return result; + + if (table->header.type != RH_TYPE_SUPER) { + vdo_free(table); + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "not a superblock region table"); + } + + result = read_super_block_data(reader, layout, table->header.payload); + if (result != UDS_SUCCESS) { + vdo_free(table); + return vdo_log_error_strerror(result, "unknown superblock format"); + } + + if (super->block_size != block_size) { + vdo_free(table); + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "superblock saved block_size %u differs from supplied block_size %zu", + super->block_size, block_size); + } + + first_block -= (super->volume_offset - super->start_offset); + result = reconstitute_layout(layout, table, first_block); + vdo_free(table); + return result; +} + +static int __must_check read_index_save_data(struct buffered_reader *reader, + struct index_save_layout *isl, + size_t saved_size) +{ + int result; + struct index_state_version file_version; + u8 buffer[sizeof(struct index_save_data) + sizeof(struct index_state_data301)]; + size_t offset = 0; + + if (saved_size != sizeof(buffer)) { + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "unexpected index save data size %zu", + saved_size); + } + + result = uds_read_from_buffered_reader(reader, buffer, sizeof(buffer)); + if (result != UDS_SUCCESS) + return vdo_log_error_strerror(result, "cannot read index save data"); + + decode_u64_le(buffer, &offset, &isl->save_data.timestamp); + decode_u64_le(buffer, &offset, &isl->save_data.nonce); + decode_u32_le(buffer, &offset, &isl->save_data.version); + offset += sizeof(u32); + + if (isl->save_data.version > 1) { + return vdo_log_error_strerror(UDS_UNSUPPORTED_VERSION, + "unknown index save version number %u", + isl->save_data.version); + } + + decode_s32_le(buffer, &offset, &file_version.signature); + decode_s32_le(buffer, &offset, &file_version.version_id); + + if ((file_version.signature != INDEX_STATE_VERSION_301.signature) || + (file_version.version_id != INDEX_STATE_VERSION_301.version_id)) { + return vdo_log_error_strerror(UDS_UNSUPPORTED_VERSION, + "index state version %d,%d is unsupported", + file_version.signature, + file_version.version_id); + } + + decode_u64_le(buffer, &offset, &isl->state_data.newest_chapter); + decode_u64_le(buffer, &offset, &isl->state_data.oldest_chapter); + decode_u64_le(buffer, &offset, &isl->state_data.last_save); + /* Skip past some historical fields that are now unused */ + offset += sizeof(u32) + sizeof(u32); + return UDS_SUCCESS; +} + +static int __must_check reconstruct_index_save(struct index_save_layout *isl, + struct region_table *table) +{ + int result; + unsigned int z; + struct layout_region *last_region; + u64 next_block = isl->index_save.start_block; + u64 last_block = next_block + isl->index_save.block_count; + + isl->zone_count = table->header.region_count - 3; + + last_region = &table->regions[table->header.region_count - 1]; + if (last_region->kind == RL_KIND_EMPTY) { + isl->free_space = *last_region; + isl->zone_count--; + } else { + isl->free_space = (struct layout_region) { + .start_block = last_block, + .block_count = 0, + .kind = RL_KIND_EMPTY, + .instance = RL_SOLE_INSTANCE, + }; + } + + isl->header = table->regions[0]; + result = verify_region(&isl->header, next_block++, RL_KIND_HEADER, + RL_SOLE_INSTANCE); + if (result != UDS_SUCCESS) + return result; + + isl->index_page_map = table->regions[1]; + result = verify_region(&isl->index_page_map, next_block, RL_KIND_INDEX_PAGE_MAP, + RL_SOLE_INSTANCE); + if (result != UDS_SUCCESS) + return result; + + next_block += isl->index_page_map.block_count; + + for (z = 0; z < isl->zone_count; z++) { + isl->volume_index_zones[z] = table->regions[z + 2]; + result = verify_region(&isl->volume_index_zones[z], next_block, + RL_KIND_VOLUME_INDEX, z); + if (result != UDS_SUCCESS) + return result; + + next_block += isl->volume_index_zones[z].block_count; + } + + isl->open_chapter = table->regions[isl->zone_count + 2]; + result = verify_region(&isl->open_chapter, next_block, RL_KIND_OPEN_CHAPTER, + RL_SOLE_INSTANCE); + if (result != UDS_SUCCESS) + return result; + + next_block += isl->open_chapter.block_count; + + result = verify_region(&isl->free_space, next_block, RL_KIND_EMPTY, + RL_SOLE_INSTANCE); + if (result != UDS_SUCCESS) + return result; + + next_block += isl->free_space.block_count; + if (next_block != last_block) { + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "index save layout table incomplete"); + } + + return UDS_SUCCESS; +} + +static int __must_check load_index_save(struct index_save_layout *isl, + struct buffered_reader *reader, + unsigned int instance) +{ + int result; + struct region_table *table = NULL; + + result = load_region_table(reader, &table); + if (result != UDS_SUCCESS) { + return vdo_log_error_strerror(result, "cannot read index save %u header", + instance); + } + + if (table->header.region_blocks != isl->index_save.block_count) { + u64 region_blocks = table->header.region_blocks; + + vdo_free(table); + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "unexpected index save %u region block count %llu", + instance, + (unsigned long long) region_blocks); + } + + if (table->header.type == RH_TYPE_UNSAVED) { + vdo_free(table); + reset_index_save_layout(isl, 0); + return UDS_SUCCESS; + } + + + if (table->header.type != RH_TYPE_SAVE) { + vdo_log_error_strerror(UDS_CORRUPT_DATA, + "unexpected index save %u header type %u", + instance, table->header.type); + vdo_free(table); + return UDS_CORRUPT_DATA; + } + + result = read_index_save_data(reader, isl, table->header.payload); + if (result != UDS_SUCCESS) { + vdo_free(table); + return vdo_log_error_strerror(result, + "unknown index save %u data format", + instance); + } + + result = reconstruct_index_save(isl, table); + vdo_free(table); + if (result != UDS_SUCCESS) { + return vdo_log_error_strerror(result, "cannot reconstruct index save %u", + instance); + } + + return UDS_SUCCESS; +} + +static int __must_check load_sub_index_regions(struct index_layout *layout) +{ + int result; + unsigned int j; + struct index_save_layout *isl; + struct buffered_reader *reader; + + for (j = 0; j < layout->super.max_saves; j++) { + isl = &layout->index.saves[j]; + result = open_region_reader(layout, &isl->index_save, &reader); + + if (result != UDS_SUCCESS) { + vdo_log_error_strerror(result, + "cannot get reader for index 0 save %u", + j); + return result; + } + + result = load_index_save(isl, reader, j); + uds_free_buffered_reader(reader); + if (result != UDS_SUCCESS) { + /* Another save slot might be valid. */ + reset_index_save_layout(isl, 0); + continue; + } + } + + return UDS_SUCCESS; +} + +static int __must_check verify_uds_index_config(struct index_layout *layout, + struct uds_configuration *config) +{ + int result; + struct buffered_reader *reader = NULL; + u64 offset; + + offset = layout->super.volume_offset - layout->super.start_offset; + result = open_layout_reader(layout, &layout->config, offset, &reader); + if (result != UDS_SUCCESS) + return vdo_log_error_strerror(result, "failed to open config reader"); + + result = uds_validate_config_contents(reader, config); + if (result != UDS_SUCCESS) { + uds_free_buffered_reader(reader); + return vdo_log_error_strerror(result, "failed to read config region"); + } + + uds_free_buffered_reader(reader); + return UDS_SUCCESS; +} + +static int load_index_layout(struct index_layout *layout, struct uds_configuration *config) +{ + int result; + struct buffered_reader *reader; + + result = uds_make_buffered_reader(layout->factory, + layout->offset / UDS_BLOCK_SIZE, 1, &reader); + if (result != UDS_SUCCESS) + return vdo_log_error_strerror(result, "unable to read superblock"); + + result = load_super_block(layout, UDS_BLOCK_SIZE, + layout->offset / UDS_BLOCK_SIZE, reader); + uds_free_buffered_reader(reader); + if (result != UDS_SUCCESS) + return result; + + result = verify_uds_index_config(layout, config); + if (result != UDS_SUCCESS) + return result; + + return load_sub_index_regions(layout); +} + +static int create_layout_factory(struct index_layout *layout, + const struct uds_configuration *config) +{ + int result; + size_t writable_size; + struct io_factory *factory = NULL; + + result = uds_make_io_factory(config->bdev, &factory); + if (result != UDS_SUCCESS) + return result; + + writable_size = uds_get_writable_size(factory) & -UDS_BLOCK_SIZE; + if (writable_size < config->size + config->offset) { + uds_put_io_factory(factory); + vdo_log_error("index storage (%zu) is smaller than the requested size %zu", + writable_size, config->size + config->offset); + return -ENOSPC; + } + + layout->factory = factory; + layout->factory_size = (config->size > 0) ? config->size : writable_size; + layout->offset = config->offset; + return UDS_SUCCESS; +} + +int uds_make_index_layout(struct uds_configuration *config, bool new_layout, + struct index_layout **layout_ptr) +{ + int result; + struct index_layout *layout = NULL; + struct save_layout_sizes sizes; + + result = compute_sizes(config, &sizes); + if (result != UDS_SUCCESS) + return result; + + result = vdo_allocate(1, struct index_layout, __func__, &layout); + if (result != VDO_SUCCESS) + return result; + + result = create_layout_factory(layout, config); + if (result != UDS_SUCCESS) { + uds_free_index_layout(layout); + return result; + } + + if (layout->factory_size < sizes.total_size) { + vdo_log_error("index storage (%zu) is smaller than the required size %llu", + layout->factory_size, + (unsigned long long) sizes.total_size); + uds_free_index_layout(layout); + return -ENOSPC; + } + + if (new_layout) + result = create_index_layout(layout, config); + else + result = load_index_layout(layout, config); + if (result != UDS_SUCCESS) { + uds_free_index_layout(layout); + return result; + } + + *layout_ptr = layout; + return UDS_SUCCESS; +} + +void uds_free_index_layout(struct index_layout *layout) +{ + if (layout == NULL) + return; + + vdo_free(layout->index.saves); + if (layout->factory != NULL) + uds_put_io_factory(layout->factory); + + vdo_free(layout); +} + +int uds_replace_index_layout_storage(struct index_layout *layout, + struct block_device *bdev) +{ + return uds_replace_storage(layout->factory, bdev); +} + +/* Obtain a dm_bufio_client for the volume region. */ +int uds_open_volume_bufio(struct index_layout *layout, size_t block_size, + unsigned int reserved_buffers, + struct dm_bufio_client **client_ptr) +{ + off_t offset = (layout->index.volume.start_block + + layout->super.volume_offset - + layout->super.start_offset); + + return uds_make_bufio(layout->factory, offset, block_size, reserved_buffers, + client_ptr); +} + +u64 uds_get_volume_nonce(struct index_layout *layout) +{ + return layout->index.nonce; +} diff --git a/drivers/md/dm-vdo/indexer/index-layout.h b/drivers/md/dm-vdo/indexer/index-layout.h new file mode 100644 index 000000000000..e9ac6f4302d6 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/index-layout.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_INDEX_LAYOUT_H +#define UDS_INDEX_LAYOUT_H + +#include "config.h" +#include "indexer.h" +#include "io-factory.h" + +/* + * The index layout describes the format of the index on the underlying storage, and is responsible + * for creating those structures when the index is first created. It also validates the index data + * when loading a saved index, and updates it when saving the index. + */ + +struct index_layout; + +int __must_check uds_make_index_layout(struct uds_configuration *config, bool new_layout, + struct index_layout **layout_ptr); + +void uds_free_index_layout(struct index_layout *layout); + +int __must_check uds_replace_index_layout_storage(struct index_layout *layout, + struct block_device *bdev); + +int __must_check uds_load_index_state(struct index_layout *layout, + struct uds_index *index); + +int __must_check uds_save_index_state(struct index_layout *layout, + struct uds_index *index); + +int __must_check uds_discard_open_chapter(struct index_layout *layout); + +u64 __must_check uds_get_volume_nonce(struct index_layout *layout); + +int __must_check uds_open_volume_bufio(struct index_layout *layout, size_t block_size, + unsigned int reserved_buffers, + struct dm_bufio_client **client_ptr); + +#endif /* UDS_INDEX_LAYOUT_H */ diff --git a/drivers/md/dm-vdo/indexer/index-page-map.c b/drivers/md/dm-vdo/indexer/index-page-map.c new file mode 100644 index 000000000000..00b44e07d0c1 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/index-page-map.c @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "index-page-map.h" + +#include "errors.h" +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "permassert.h" +#include "string-utils.h" +#include "thread-utils.h" + +#include "hash-utils.h" +#include "indexer.h" + +/* + * The index page map is conceptually a two-dimensional array indexed by chapter number and index + * page number within the chapter. Each entry contains the number of the last delta list on that + * index page. In order to save memory, the information for the last page in each chapter is not + * recorded, as it is known from the geometry. + */ + +static const u8 PAGE_MAP_MAGIC[] = "ALBIPM02"; + +#define PAGE_MAP_MAGIC_LENGTH (sizeof(PAGE_MAP_MAGIC) - 1) + +static inline u32 get_entry_count(const struct index_geometry *geometry) +{ + return geometry->chapters_per_volume * (geometry->index_pages_per_chapter - 1); +} + +int uds_make_index_page_map(const struct index_geometry *geometry, + struct index_page_map **map_ptr) +{ + int result; + struct index_page_map *map; + + result = vdo_allocate(1, struct index_page_map, "page map", &map); + if (result != VDO_SUCCESS) + return result; + + map->geometry = geometry; + map->entries_per_chapter = geometry->index_pages_per_chapter - 1; + result = vdo_allocate(get_entry_count(geometry), u16, "Index Page Map Entries", + &map->entries); + if (result != VDO_SUCCESS) { + uds_free_index_page_map(map); + return result; + } + + *map_ptr = map; + return UDS_SUCCESS; +} + +void uds_free_index_page_map(struct index_page_map *map) +{ + if (map != NULL) { + vdo_free(map->entries); + vdo_free(map); + } +} + +void uds_update_index_page_map(struct index_page_map *map, u64 virtual_chapter_number, + u32 chapter_number, u32 index_page_number, + u32 delta_list_number) +{ + size_t slot; + + map->last_update = virtual_chapter_number; + if (index_page_number == map->entries_per_chapter) + return; + + slot = (chapter_number * map->entries_per_chapter) + index_page_number; + map->entries[slot] = delta_list_number; +} + +u32 uds_find_index_page_number(const struct index_page_map *map, + const struct uds_record_name *name, u32 chapter_number) +{ + u32 delta_list_number = uds_hash_to_chapter_delta_list(name, map->geometry); + u32 slot = chapter_number * map->entries_per_chapter; + u32 page; + + for (page = 0; page < map->entries_per_chapter; page++) { + if (delta_list_number <= map->entries[slot + page]) + break; + } + + return page; +} + +void uds_get_list_number_bounds(const struct index_page_map *map, u32 chapter_number, + u32 index_page_number, u32 *lowest_list, + u32 *highest_list) +{ + u32 slot = chapter_number * map->entries_per_chapter; + + *lowest_list = ((index_page_number == 0) ? + 0 : map->entries[slot + index_page_number - 1] + 1); + *highest_list = ((index_page_number < map->entries_per_chapter) ? + map->entries[slot + index_page_number] : + map->geometry->delta_lists_per_chapter - 1); +} + +u64 uds_compute_index_page_map_save_size(const struct index_geometry *geometry) +{ + return PAGE_MAP_MAGIC_LENGTH + sizeof(u64) + sizeof(u16) * get_entry_count(geometry); +} + +int uds_write_index_page_map(struct index_page_map *map, struct buffered_writer *writer) +{ + int result; + u8 *buffer; + size_t offset = 0; + u64 saved_size = uds_compute_index_page_map_save_size(map->geometry); + u32 i; + + result = vdo_allocate(saved_size, u8, "page map data", &buffer); + if (result != VDO_SUCCESS) + return result; + + memcpy(buffer, PAGE_MAP_MAGIC, PAGE_MAP_MAGIC_LENGTH); + offset += PAGE_MAP_MAGIC_LENGTH; + encode_u64_le(buffer, &offset, map->last_update); + for (i = 0; i < get_entry_count(map->geometry); i++) + encode_u16_le(buffer, &offset, map->entries[i]); + + result = uds_write_to_buffered_writer(writer, buffer, offset); + vdo_free(buffer); + if (result != UDS_SUCCESS) + return result; + + return uds_flush_buffered_writer(writer); +} + +int uds_read_index_page_map(struct index_page_map *map, struct buffered_reader *reader) +{ + int result; + u8 magic[PAGE_MAP_MAGIC_LENGTH]; + u8 *buffer; + size_t offset = 0; + u64 saved_size = uds_compute_index_page_map_save_size(map->geometry); + u32 i; + + result = vdo_allocate(saved_size, u8, "page map data", &buffer); + if (result != VDO_SUCCESS) + return result; + + result = uds_read_from_buffered_reader(reader, buffer, saved_size); + if (result != UDS_SUCCESS) { + vdo_free(buffer); + return result; + } + + memcpy(&magic, buffer, PAGE_MAP_MAGIC_LENGTH); + offset += PAGE_MAP_MAGIC_LENGTH; + if (memcmp(magic, PAGE_MAP_MAGIC, PAGE_MAP_MAGIC_LENGTH) != 0) { + vdo_free(buffer); + return UDS_CORRUPT_DATA; + } + + decode_u64_le(buffer, &offset, &map->last_update); + for (i = 0; i < get_entry_count(map->geometry); i++) + decode_u16_le(buffer, &offset, &map->entries[i]); + + vdo_free(buffer); + vdo_log_debug("read index page map, last update %llu", + (unsigned long long) map->last_update); + return UDS_SUCCESS; +} diff --git a/drivers/md/dm-vdo/indexer/index-page-map.h b/drivers/md/dm-vdo/indexer/index-page-map.h new file mode 100644 index 000000000000..b327c0bb9656 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/index-page-map.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_INDEX_PAGE_MAP_H +#define UDS_INDEX_PAGE_MAP_H + +#include "geometry.h" +#include "io-factory.h" + +/* + * The index maintains a page map which records how the chapter delta lists are distributed among + * the index pages for each chapter, allowing the volume to be efficient about reading only pages + * that it knows it will need. + */ + +struct index_page_map { + const struct index_geometry *geometry; + u64 last_update; + u32 entries_per_chapter; + u16 *entries; +}; + +int __must_check uds_make_index_page_map(const struct index_geometry *geometry, + struct index_page_map **map_ptr); + +void uds_free_index_page_map(struct index_page_map *map); + +int __must_check uds_read_index_page_map(struct index_page_map *map, + struct buffered_reader *reader); + +int __must_check uds_write_index_page_map(struct index_page_map *map, + struct buffered_writer *writer); + +void uds_update_index_page_map(struct index_page_map *map, u64 virtual_chapter_number, + u32 chapter_number, u32 index_page_number, + u32 delta_list_number); + +u32 __must_check uds_find_index_page_number(const struct index_page_map *map, + const struct uds_record_name *name, + u32 chapter_number); + +void uds_get_list_number_bounds(const struct index_page_map *map, u32 chapter_number, + u32 index_page_number, u32 *lowest_list, + u32 *highest_list); + +u64 uds_compute_index_page_map_save_size(const struct index_geometry *geometry); + +#endif /* UDS_INDEX_PAGE_MAP_H */ diff --git a/drivers/md/dm-vdo/indexer/index-session.c b/drivers/md/dm-vdo/indexer/index-session.c new file mode 100644 index 000000000000..aee0914d604a --- /dev/null +++ b/drivers/md/dm-vdo/indexer/index-session.c @@ -0,0 +1,739 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "index-session.h" + +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "time-utils.h" + +#include "funnel-requestqueue.h" +#include "index.h" +#include "index-layout.h" + +/* + * The index session contains a lock (the request_mutex) which ensures that only one thread can + * change the state of its index at a time. The state field indicates the current state of the + * index through a set of descriptive flags. The request_mutex must be notified whenever a + * non-transient state flag is cleared. The request_mutex is also used to count the number of + * requests currently in progress so that they can be drained when suspending or closing the index. + * + * If the index session is suspended shortly after opening an index, it may have to suspend during + * a rebuild. Depending on the size of the index, a rebuild may take a significant amount of time, + * so UDS allows the rebuild to be paused in order to suspend the session in a timely manner. When + * the index session is resumed, the rebuild can continue from where it left off. If the index + * session is shut down with a suspended rebuild, the rebuild progress is abandoned and the rebuild + * will start from the beginning the next time the index is loaded. The mutex and status fields in + * the index_load_context are used to record the state of any interrupted rebuild. + */ + +enum index_session_flag_bit { + IS_FLAG_BIT_START = 8, + /* The session has started loading an index but not completed it. */ + IS_FLAG_BIT_LOADING = IS_FLAG_BIT_START, + /* The session has loaded an index, which can handle requests. */ + IS_FLAG_BIT_LOADED, + /* The session's index has been permanently disabled. */ + IS_FLAG_BIT_DISABLED, + /* The session's index is suspended. */ + IS_FLAG_BIT_SUSPENDED, + /* The session is handling some index state change. */ + IS_FLAG_BIT_WAITING, + /* The session's index is closing and draining requests. */ + IS_FLAG_BIT_CLOSING, + /* The session is being destroyed and is draining requests. */ + IS_FLAG_BIT_DESTROYING, +}; + +enum index_session_flag { + IS_FLAG_LOADED = (1 << IS_FLAG_BIT_LOADED), + IS_FLAG_LOADING = (1 << IS_FLAG_BIT_LOADING), + IS_FLAG_DISABLED = (1 << IS_FLAG_BIT_DISABLED), + IS_FLAG_SUSPENDED = (1 << IS_FLAG_BIT_SUSPENDED), + IS_FLAG_WAITING = (1 << IS_FLAG_BIT_WAITING), + IS_FLAG_CLOSING = (1 << IS_FLAG_BIT_CLOSING), + IS_FLAG_DESTROYING = (1 << IS_FLAG_BIT_DESTROYING), +}; + +/* Release a reference to an index session. */ +static void release_index_session(struct uds_index_session *index_session) +{ + mutex_lock(&index_session->request_mutex); + if (--index_session->request_count == 0) + uds_broadcast_cond(&index_session->request_cond); + mutex_unlock(&index_session->request_mutex); +} + +/* + * Acquire a reference to the index session for an asynchronous index request. The reference must + * eventually be released with a corresponding call to release_index_session(). + */ +static int get_index_session(struct uds_index_session *index_session) +{ + unsigned int state; + int result = UDS_SUCCESS; + + mutex_lock(&index_session->request_mutex); + index_session->request_count++; + state = index_session->state; + mutex_unlock(&index_session->request_mutex); + + if (state == IS_FLAG_LOADED) { + return UDS_SUCCESS; + } else if (state & IS_FLAG_DISABLED) { + result = UDS_DISABLED; + } else if ((state & IS_FLAG_LOADING) || + (state & IS_FLAG_SUSPENDED) || + (state & IS_FLAG_WAITING)) { + result = -EBUSY; + } else { + result = UDS_NO_INDEX; + } + + release_index_session(index_session); + return result; +} + +int uds_launch_request(struct uds_request *request) +{ + size_t internal_size; + int result; + + if (request->callback == NULL) { + vdo_log_error("missing required callback"); + return -EINVAL; + } + + switch (request->type) { + case UDS_DELETE: + case UDS_POST: + case UDS_QUERY: + case UDS_QUERY_NO_UPDATE: + case UDS_UPDATE: + break; + default: + vdo_log_error("received invalid callback type"); + return -EINVAL; + } + + /* Reset all internal fields before processing. */ + internal_size = + sizeof(struct uds_request) - offsetof(struct uds_request, zone_number); + // FIXME should be using struct_group for this instead + memset((char *) request + sizeof(*request) - internal_size, 0, internal_size); + + result = get_index_session(request->session); + if (result != UDS_SUCCESS) + return result; + + request->found = false; + request->unbatched = false; + request->index = request->session->index; + + uds_enqueue_request(request, STAGE_TRIAGE); + return UDS_SUCCESS; +} + +static void enter_callback_stage(struct uds_request *request) +{ + if (request->status != UDS_SUCCESS) { + /* All request errors are considered unrecoverable */ + mutex_lock(&request->session->request_mutex); + request->session->state |= IS_FLAG_DISABLED; + mutex_unlock(&request->session->request_mutex); + } + + uds_request_queue_enqueue(request->session->callback_queue, request); +} + +static inline void count_once(u64 *count_ptr) +{ + WRITE_ONCE(*count_ptr, READ_ONCE(*count_ptr) + 1); +} + +static void update_session_stats(struct uds_request *request) +{ + struct session_stats *session_stats = &request->session->stats; + + count_once(&session_stats->requests); + + switch (request->type) { + case UDS_POST: + if (request->found) + count_once(&session_stats->posts_found); + else + count_once(&session_stats->posts_not_found); + + if (request->location == UDS_LOCATION_IN_OPEN_CHAPTER) + count_once(&session_stats->posts_found_open_chapter); + else if (request->location == UDS_LOCATION_IN_DENSE) + count_once(&session_stats->posts_found_dense); + else if (request->location == UDS_LOCATION_IN_SPARSE) + count_once(&session_stats->posts_found_sparse); + break; + + case UDS_UPDATE: + if (request->found) + count_once(&session_stats->updates_found); + else + count_once(&session_stats->updates_not_found); + break; + + case UDS_DELETE: + if (request->found) + count_once(&session_stats->deletions_found); + else + count_once(&session_stats->deletions_not_found); + break; + + case UDS_QUERY: + case UDS_QUERY_NO_UPDATE: + if (request->found) + count_once(&session_stats->queries_found); + else + count_once(&session_stats->queries_not_found); + break; + + default: + request->status = VDO_ASSERT(false, "unknown request type: %d", + request->type); + } +} + +static void handle_callbacks(struct uds_request *request) +{ + struct uds_index_session *index_session = request->session; + + if (request->status == UDS_SUCCESS) + update_session_stats(request); + + request->status = uds_status_to_errno(request->status); + request->callback(request); + release_index_session(index_session); +} + +static int __must_check make_empty_index_session(struct uds_index_session **index_session_ptr) +{ + int result; + struct uds_index_session *session; + + result = vdo_allocate(1, struct uds_index_session, __func__, &session); + if (result != VDO_SUCCESS) + return result; + + mutex_init(&session->request_mutex); + uds_init_cond(&session->request_cond); + mutex_init(&session->load_context.mutex); + uds_init_cond(&session->load_context.cond); + + result = uds_make_request_queue("callbackW", &handle_callbacks, + &session->callback_queue); + if (result != UDS_SUCCESS) { + vdo_free(session); + return result; + } + + *index_session_ptr = session; + return UDS_SUCCESS; +} + +int uds_create_index_session(struct uds_index_session **session) +{ + if (session == NULL) { + vdo_log_error("missing session pointer"); + return -EINVAL; + } + + return uds_status_to_errno(make_empty_index_session(session)); +} + +static int __must_check start_loading_index_session(struct uds_index_session *index_session) +{ + int result; + + mutex_lock(&index_session->request_mutex); + if (index_session->state & IS_FLAG_SUSPENDED) { + vdo_log_info("Index session is suspended"); + result = -EBUSY; + } else if (index_session->state != 0) { + vdo_log_info("Index is already loaded"); + result = -EBUSY; + } else { + index_session->state |= IS_FLAG_LOADING; + result = UDS_SUCCESS; + } + mutex_unlock(&index_session->request_mutex); + return result; +} + +static void finish_loading_index_session(struct uds_index_session *index_session, + int result) +{ + mutex_lock(&index_session->request_mutex); + index_session->state &= ~IS_FLAG_LOADING; + if (result == UDS_SUCCESS) + index_session->state |= IS_FLAG_LOADED; + + uds_broadcast_cond(&index_session->request_cond); + mutex_unlock(&index_session->request_mutex); +} + +static int initialize_index_session(struct uds_index_session *index_session, + enum uds_open_index_type open_type) +{ + int result; + struct uds_configuration *config; + + result = uds_make_configuration(&index_session->parameters, &config); + if (result != UDS_SUCCESS) { + vdo_log_error_strerror(result, "Failed to allocate config"); + return result; + } + + memset(&index_session->stats, 0, sizeof(index_session->stats)); + result = uds_make_index(config, open_type, &index_session->load_context, + enter_callback_stage, &index_session->index); + if (result != UDS_SUCCESS) + vdo_log_error_strerror(result, "Failed to make index"); + else + uds_log_configuration(config); + + uds_free_configuration(config); + return result; +} + +static const char *get_open_type_string(enum uds_open_index_type open_type) +{ + switch (open_type) { + case UDS_CREATE: + return "creating index"; + case UDS_LOAD: + return "loading or rebuilding index"; + case UDS_NO_REBUILD: + return "loading index"; + default: + return "unknown open method"; + } +} + +/* + * Open an index under the given session. This operation will fail if the + * index session is suspended, or if there is already an open index. + */ +int uds_open_index(enum uds_open_index_type open_type, + const struct uds_parameters *parameters, + struct uds_index_session *session) +{ + int result; + char name[BDEVNAME_SIZE]; + + if (parameters == NULL) { + vdo_log_error("missing required parameters"); + return -EINVAL; + } + if (parameters->bdev == NULL) { + vdo_log_error("missing required block device"); + return -EINVAL; + } + if (session == NULL) { + vdo_log_error("missing required session pointer"); + return -EINVAL; + } + + result = start_loading_index_session(session); + if (result != UDS_SUCCESS) + return uds_status_to_errno(result); + + session->parameters = *parameters; + format_dev_t(name, parameters->bdev->bd_dev); + vdo_log_info("%s: %s", get_open_type_string(open_type), name); + + result = initialize_index_session(session, open_type); + if (result != UDS_SUCCESS) + vdo_log_error_strerror(result, "Failed %s", + get_open_type_string(open_type)); + + finish_loading_index_session(session, result); + return uds_status_to_errno(result); +} + +static void wait_for_no_requests_in_progress(struct uds_index_session *index_session) +{ + mutex_lock(&index_session->request_mutex); + while (index_session->request_count > 0) { + uds_wait_cond(&index_session->request_cond, + &index_session->request_mutex); + } + mutex_unlock(&index_session->request_mutex); +} + +static int __must_check save_index(struct uds_index_session *index_session) +{ + wait_for_no_requests_in_progress(index_session); + return uds_save_index(index_session->index); +} + +static void suspend_rebuild(struct uds_index_session *session) +{ + mutex_lock(&session->load_context.mutex); + switch (session->load_context.status) { + case INDEX_OPENING: + session->load_context.status = INDEX_SUSPENDING; + + /* Wait until the index indicates that it is not replaying. */ + while ((session->load_context.status != INDEX_SUSPENDED) && + (session->load_context.status != INDEX_READY)) { + uds_wait_cond(&session->load_context.cond, + &session->load_context.mutex); + } + + break; + + case INDEX_READY: + /* Index load does not need to be suspended. */ + break; + + case INDEX_SUSPENDED: + case INDEX_SUSPENDING: + case INDEX_FREEING: + default: + /* These cases should not happen. */ + VDO_ASSERT_LOG_ONLY(false, "Bad load context state %u", + session->load_context.status); + break; + } + mutex_unlock(&session->load_context.mutex); +} + +/* + * Suspend index operation, draining all current index requests and preventing new index requests + * from starting. Optionally saves all index data before returning. + */ +int uds_suspend_index_session(struct uds_index_session *session, bool save) +{ + int result = UDS_SUCCESS; + bool no_work = false; + bool rebuilding = false; + + /* Wait for any current index state change to complete. */ + mutex_lock(&session->request_mutex); + while (session->state & IS_FLAG_CLOSING) + uds_wait_cond(&session->request_cond, &session->request_mutex); + + if ((session->state & IS_FLAG_WAITING) || (session->state & IS_FLAG_DESTROYING)) { + no_work = true; + vdo_log_info("Index session is already changing state"); + result = -EBUSY; + } else if (session->state & IS_FLAG_SUSPENDED) { + no_work = true; + } else if (session->state & IS_FLAG_LOADING) { + session->state |= IS_FLAG_WAITING; + rebuilding = true; + } else if (session->state & IS_FLAG_LOADED) { + session->state |= IS_FLAG_WAITING; + } else { + no_work = true; + session->state |= IS_FLAG_SUSPENDED; + uds_broadcast_cond(&session->request_cond); + } + mutex_unlock(&session->request_mutex); + + if (no_work) + return uds_status_to_errno(result); + + if (rebuilding) + suspend_rebuild(session); + else if (save) + result = save_index(session); + else + result = uds_flush_index_session(session); + + mutex_lock(&session->request_mutex); + session->state &= ~IS_FLAG_WAITING; + session->state |= IS_FLAG_SUSPENDED; + uds_broadcast_cond(&session->request_cond); + mutex_unlock(&session->request_mutex); + return uds_status_to_errno(result); +} + +static int replace_device(struct uds_index_session *session, struct block_device *bdev) +{ + int result; + + result = uds_replace_index_storage(session->index, bdev); + if (result != UDS_SUCCESS) + return result; + + session->parameters.bdev = bdev; + return UDS_SUCCESS; +} + +/* + * Resume index operation after being suspended. If the index is suspended and the supplied block + * device differs from the current backing store, the index will start using the new backing store. + */ +int uds_resume_index_session(struct uds_index_session *session, + struct block_device *bdev) +{ + int result = UDS_SUCCESS; + bool no_work = false; + bool resume_replay = false; + + mutex_lock(&session->request_mutex); + if (session->state & IS_FLAG_WAITING) { + vdo_log_info("Index session is already changing state"); + no_work = true; + result = -EBUSY; + } else if (!(session->state & IS_FLAG_SUSPENDED)) { + /* If not suspended, just succeed. */ + no_work = true; + result = UDS_SUCCESS; + } else { + session->state |= IS_FLAG_WAITING; + if (session->state & IS_FLAG_LOADING) + resume_replay = true; + } + mutex_unlock(&session->request_mutex); + + if (no_work) + return result; + + if ((session->index != NULL) && (bdev != session->parameters.bdev)) { + result = replace_device(session, bdev); + if (result != UDS_SUCCESS) { + mutex_lock(&session->request_mutex); + session->state &= ~IS_FLAG_WAITING; + uds_broadcast_cond(&session->request_cond); + mutex_unlock(&session->request_mutex); + return uds_status_to_errno(result); + } + } + + if (resume_replay) { + mutex_lock(&session->load_context.mutex); + switch (session->load_context.status) { + case INDEX_SUSPENDED: + session->load_context.status = INDEX_OPENING; + /* Notify the index to start replaying again. */ + uds_broadcast_cond(&session->load_context.cond); + break; + + case INDEX_READY: + /* There is no index rebuild to resume. */ + break; + + case INDEX_OPENING: + case INDEX_SUSPENDING: + case INDEX_FREEING: + default: + /* These cases should not happen; do nothing. */ + VDO_ASSERT_LOG_ONLY(false, "Bad load context state %u", + session->load_context.status); + break; + } + mutex_unlock(&session->load_context.mutex); + } + + mutex_lock(&session->request_mutex); + session->state &= ~IS_FLAG_WAITING; + session->state &= ~IS_FLAG_SUSPENDED; + uds_broadcast_cond(&session->request_cond); + mutex_unlock(&session->request_mutex); + return UDS_SUCCESS; +} + +static int save_and_free_index(struct uds_index_session *index_session) +{ + int result = UDS_SUCCESS; + bool suspended; + struct uds_index *index = index_session->index; + + if (index == NULL) + return UDS_SUCCESS; + + mutex_lock(&index_session->request_mutex); + suspended = (index_session->state & IS_FLAG_SUSPENDED); + mutex_unlock(&index_session->request_mutex); + + if (!suspended) { + result = uds_save_index(index); + if (result != UDS_SUCCESS) + vdo_log_warning_strerror(result, + "ignoring error from save_index"); + } + uds_free_index(index); + index_session->index = NULL; + + /* + * Reset all index state that happens to be in the index + * session, so it doesn't affect any future index. + */ + mutex_lock(&index_session->load_context.mutex); + index_session->load_context.status = INDEX_OPENING; + mutex_unlock(&index_session->load_context.mutex); + + mutex_lock(&index_session->request_mutex); + /* Only the suspend bit will remain relevant. */ + index_session->state &= IS_FLAG_SUSPENDED; + mutex_unlock(&index_session->request_mutex); + + return result; +} + +/* Save and close the current index. */ +int uds_close_index(struct uds_index_session *index_session) +{ + int result = UDS_SUCCESS; + + /* Wait for any current index state change to complete. */ + mutex_lock(&index_session->request_mutex); + while ((index_session->state & IS_FLAG_WAITING) || + (index_session->state & IS_FLAG_CLOSING)) { + uds_wait_cond(&index_session->request_cond, + &index_session->request_mutex); + } + + if (index_session->state & IS_FLAG_SUSPENDED) { + vdo_log_info("Index session is suspended"); + result = -EBUSY; + } else if ((index_session->state & IS_FLAG_DESTROYING) || + !(index_session->state & IS_FLAG_LOADED)) { + /* The index doesn't exist, hasn't finished loading, or is being destroyed. */ + result = UDS_NO_INDEX; + } else { + index_session->state |= IS_FLAG_CLOSING; + } + mutex_unlock(&index_session->request_mutex); + if (result != UDS_SUCCESS) + return uds_status_to_errno(result); + + vdo_log_debug("Closing index"); + wait_for_no_requests_in_progress(index_session); + result = save_and_free_index(index_session); + vdo_log_debug("Closed index"); + + mutex_lock(&index_session->request_mutex); + index_session->state &= ~IS_FLAG_CLOSING; + uds_broadcast_cond(&index_session->request_cond); + mutex_unlock(&index_session->request_mutex); + return uds_status_to_errno(result); +} + +/* This will save and close an open index before destroying the session. */ +int uds_destroy_index_session(struct uds_index_session *index_session) +{ + int result; + bool load_pending = false; + + vdo_log_debug("Destroying index session"); + + /* Wait for any current index state change to complete. */ + mutex_lock(&index_session->request_mutex); + while ((index_session->state & IS_FLAG_WAITING) || + (index_session->state & IS_FLAG_CLOSING)) { + uds_wait_cond(&index_session->request_cond, + &index_session->request_mutex); + } + + if (index_session->state & IS_FLAG_DESTROYING) { + mutex_unlock(&index_session->request_mutex); + vdo_log_info("Index session is already closing"); + return -EBUSY; + } + + index_session->state |= IS_FLAG_DESTROYING; + load_pending = ((index_session->state & IS_FLAG_LOADING) && + (index_session->state & IS_FLAG_SUSPENDED)); + mutex_unlock(&index_session->request_mutex); + + if (load_pending) { + /* Tell the index to terminate the rebuild. */ + mutex_lock(&index_session->load_context.mutex); + if (index_session->load_context.status == INDEX_SUSPENDED) { + index_session->load_context.status = INDEX_FREEING; + uds_broadcast_cond(&index_session->load_context.cond); + } + mutex_unlock(&index_session->load_context.mutex); + + /* Wait until the load exits before proceeding. */ + mutex_lock(&index_session->request_mutex); + while (index_session->state & IS_FLAG_LOADING) { + uds_wait_cond(&index_session->request_cond, + &index_session->request_mutex); + } + mutex_unlock(&index_session->request_mutex); + } + + wait_for_no_requests_in_progress(index_session); + result = save_and_free_index(index_session); + uds_request_queue_finish(index_session->callback_queue); + index_session->callback_queue = NULL; + vdo_log_debug("Destroyed index session"); + vdo_free(index_session); + return uds_status_to_errno(result); +} + +/* Wait until all callbacks for index operations are complete. */ +int uds_flush_index_session(struct uds_index_session *index_session) +{ + wait_for_no_requests_in_progress(index_session); + uds_wait_for_idle_index(index_session->index); + return UDS_SUCCESS; +} + +/* Statistics collection is intended to be thread-safe. */ +static void collect_stats(const struct uds_index_session *index_session, + struct uds_index_stats *stats) +{ + const struct session_stats *session_stats = &index_session->stats; + + stats->current_time = ktime_to_seconds(current_time_ns(CLOCK_REALTIME)); + stats->posts_found = READ_ONCE(session_stats->posts_found); + stats->in_memory_posts_found = READ_ONCE(session_stats->posts_found_open_chapter); + stats->dense_posts_found = READ_ONCE(session_stats->posts_found_dense); + stats->sparse_posts_found = READ_ONCE(session_stats->posts_found_sparse); + stats->posts_not_found = READ_ONCE(session_stats->posts_not_found); + stats->updates_found = READ_ONCE(session_stats->updates_found); + stats->updates_not_found = READ_ONCE(session_stats->updates_not_found); + stats->deletions_found = READ_ONCE(session_stats->deletions_found); + stats->deletions_not_found = READ_ONCE(session_stats->deletions_not_found); + stats->queries_found = READ_ONCE(session_stats->queries_found); + stats->queries_not_found = READ_ONCE(session_stats->queries_not_found); + stats->requests = READ_ONCE(session_stats->requests); +} + +int uds_get_index_session_stats(struct uds_index_session *index_session, + struct uds_index_stats *stats) +{ + if (stats == NULL) { + vdo_log_error("received a NULL index stats pointer"); + return -EINVAL; + } + + collect_stats(index_session, stats); + if (index_session->index != NULL) { + uds_get_index_stats(index_session->index, stats); + } else { + stats->entries_indexed = 0; + stats->memory_used = 0; + stats->collisions = 0; + stats->entries_discarded = 0; + } + + return UDS_SUCCESS; +} + +void uds_wait_cond(struct cond_var *cv, struct mutex *mutex) +{ + DEFINE_WAIT(__wait); + + prepare_to_wait(&cv->wait_queue, &__wait, TASK_IDLE); + mutex_unlock(mutex); + schedule(); + finish_wait(&cv->wait_queue, &__wait); + mutex_lock(mutex); +} diff --git a/drivers/md/dm-vdo/indexer/index-session.h b/drivers/md/dm-vdo/indexer/index-session.h new file mode 100644 index 000000000000..066648f6e062 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/index-session.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_INDEX_SESSION_H +#define UDS_INDEX_SESSION_H + +#include +#include + +#include "thread-utils.h" + +#include "config.h" +#include "indexer.h" + +/* + * The index session mediates all interactions with a UDS index. Once the index session is created, + * it can be used to open, close, suspend, or recreate an index. It implements the majority of the + * functions in the top-level UDS API. + * + * If any deduplication request fails due to an internal error, the index is marked disabled. It + * will not accept any further requests and can only be closed. Closing the index will clear the + * disabled flag, and the index can then be reopened and recovered using the same index session. + */ + +struct __aligned(L1_CACHE_BYTES) session_stats { + /* Post requests that found an entry */ + u64 posts_found; + /* Post requests found in the open chapter */ + u64 posts_found_open_chapter; + /* Post requests found in the dense index */ + u64 posts_found_dense; + /* Post requests found in the sparse index */ + u64 posts_found_sparse; + /* Post requests that did not find an entry */ + u64 posts_not_found; + /* Update requests that found an entry */ + u64 updates_found; + /* Update requests that did not find an entry */ + u64 updates_not_found; + /* Delete requests that found an entry */ + u64 deletions_found; + /* Delete requests that did not find an entry */ + u64 deletions_not_found; + /* Query requests that found an entry */ + u64 queries_found; + /* Query requests that did not find an entry */ + u64 queries_not_found; + /* Total number of requests */ + u64 requests; +}; + +enum index_suspend_status { + /* An index load has started but the index is not ready for use. */ + INDEX_OPENING = 0, + /* The index is able to handle requests. */ + INDEX_READY, + /* The index is attempting to suspend a rebuild. */ + INDEX_SUSPENDING, + /* An index rebuild has been suspended. */ + INDEX_SUSPENDED, + /* An index rebuild is being stopped in order to shut down. */ + INDEX_FREEING, +}; + +struct index_load_context { + struct mutex mutex; + struct cond_var cond; + enum index_suspend_status status; +}; + +struct uds_index_session { + unsigned int state; + struct uds_index *index; + struct uds_request_queue *callback_queue; + struct uds_parameters parameters; + struct index_load_context load_context; + struct mutex request_mutex; + struct cond_var request_cond; + int request_count; + struct session_stats stats; +}; + +#endif /* UDS_INDEX_SESSION_H */ diff --git a/drivers/md/dm-vdo/indexer/index.c b/drivers/md/dm-vdo/indexer/index.c new file mode 100644 index 000000000000..1ba767144426 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/index.c @@ -0,0 +1,1388 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + + +#include "index.h" + +#include "logger.h" +#include "memory-alloc.h" + +#include "funnel-requestqueue.h" +#include "hash-utils.h" +#include "sparse-cache.h" + +static const u64 NO_LAST_SAVE = U64_MAX; + +/* + * When searching for deduplication records, the index first searches the volume index, and then + * searches the chapter index for the relevant chapter. If the chapter has been fully committed to + * storage, the chapter pages are loaded into the page cache. If the chapter has not yet been + * committed (either the open chapter or a recently closed one), the index searches the in-memory + * representation of the chapter. Finally, if the volume index does not find a record and the index + * is sparse, the index will search the sparse cache. + * + * The index send two kinds of messages to coordinate between zones: chapter close messages for the + * chapter writer, and sparse cache barrier messages for the sparse cache. + * + * The chapter writer is responsible for committing chapters of records to storage. Since zones can + * get different numbers of records, some zones may fall behind others. Each time a zone fills up + * its available space in a chapter, it informs the chapter writer that the chapter is complete, + * and also informs all other zones that it has closed the chapter. Each other zone will then close + * the chapter immediately, regardless of how full it is, in order to minimize skew between zones. + * Once every zone has closed the chapter, the chapter writer will commit that chapter to storage. + * + * The last zone to close the chapter also removes the oldest chapter from the volume index. + * Although that chapter is invalid for zones that have moved on, the existence of the open chapter + * means that those zones will never ask the volume index about it. No zone is allowed to get more + * than one chapter ahead of any other. If a zone is so far ahead that it tries to close another + * chapter before the previous one has been closed by all zones, it is forced to wait. + * + * The sparse cache relies on having the same set of chapter indexes available to all zones. When a + * request wants to add a chapter to the sparse cache, it sends a barrier message to each zone + * during the triage stage that acts as a rendezvous. Once every zone has reached the barrier and + * paused its operations, the cache membership is changed and each zone is then informed that it + * can proceed. More details can be found in the sparse cache documentation. + * + * If a sparse cache has only one zone, it will not create a triage queue, but it still needs the + * barrier message to change the sparse cache membership, so the index simulates the message by + * invoking the handler directly. + */ + +struct chapter_writer { + /* The index to which we belong */ + struct uds_index *index; + /* The thread to do the writing */ + struct thread *thread; + /* The lock protecting the following fields */ + struct mutex mutex; + /* The condition signalled on state changes */ + struct cond_var cond; + /* Set to true to stop the thread */ + bool stop; + /* The result from the most recent write */ + int result; + /* The number of bytes allocated by the chapter writer */ + size_t memory_size; + /* The number of zones which have submitted a chapter for writing */ + unsigned int zones_to_write; + /* Open chapter index used by uds_close_open_chapter() */ + struct open_chapter_index *open_chapter_index; + /* Collated records used by uds_close_open_chapter() */ + struct uds_volume_record *collated_records; + /* The chapters to write (one per zone) */ + struct open_chapter_zone *chapters[]; +}; + +static bool is_zone_chapter_sparse(const struct index_zone *zone, u64 virtual_chapter) +{ + return uds_is_chapter_sparse(zone->index->volume->geometry, + zone->oldest_virtual_chapter, + zone->newest_virtual_chapter, virtual_chapter); +} + +static int launch_zone_message(struct uds_zone_message message, unsigned int zone, + struct uds_index *index) +{ + int result; + struct uds_request *request; + + result = vdo_allocate(1, struct uds_request, __func__, &request); + if (result != VDO_SUCCESS) + return result; + + request->index = index; + request->unbatched = true; + request->zone_number = zone; + request->zone_message = message; + + uds_enqueue_request(request, STAGE_MESSAGE); + return UDS_SUCCESS; +} + +static void enqueue_barrier_messages(struct uds_index *index, u64 virtual_chapter) +{ + struct uds_zone_message message = { + .type = UDS_MESSAGE_SPARSE_CACHE_BARRIER, + .virtual_chapter = virtual_chapter, + }; + unsigned int zone; + + for (zone = 0; zone < index->zone_count; zone++) { + int result = launch_zone_message(message, zone, index); + + VDO_ASSERT_LOG_ONLY((result == UDS_SUCCESS), "barrier message allocation"); + } +} + +/* + * Determine whether this request should trigger a sparse cache barrier message to change the + * membership of the sparse cache. If a change in membership is desired, the function returns the + * chapter number to add. + */ +static u64 triage_index_request(struct uds_index *index, struct uds_request *request) +{ + u64 virtual_chapter; + struct index_zone *zone; + + virtual_chapter = uds_lookup_volume_index_name(index->volume_index, + &request->record_name); + if (virtual_chapter == NO_CHAPTER) + return NO_CHAPTER; + + zone = index->zones[request->zone_number]; + if (!is_zone_chapter_sparse(zone, virtual_chapter)) + return NO_CHAPTER; + + /* + * FIXME: Optimize for a common case by remembering the chapter from the most recent + * barrier message and skipping this chapter if is it the same. + */ + + return virtual_chapter; +} + +/* + * Simulate a message to change the sparse cache membership for a single-zone sparse index. This + * allows us to forgo the complicated locking required by a multi-zone sparse index. Any other kind + * of index does nothing here. + */ +static int simulate_index_zone_barrier_message(struct index_zone *zone, + struct uds_request *request) +{ + u64 sparse_virtual_chapter; + + if ((zone->index->zone_count > 1) || + !uds_is_sparse_index_geometry(zone->index->volume->geometry)) + return UDS_SUCCESS; + + sparse_virtual_chapter = triage_index_request(zone->index, request); + if (sparse_virtual_chapter == NO_CHAPTER) + return UDS_SUCCESS; + + return uds_update_sparse_cache(zone, sparse_virtual_chapter); +} + +/* This is the request processing function for the triage queue. */ +static void triage_request(struct uds_request *request) +{ + struct uds_index *index = request->index; + u64 sparse_virtual_chapter = triage_index_request(index, request); + + if (sparse_virtual_chapter != NO_CHAPTER) + enqueue_barrier_messages(index, sparse_virtual_chapter); + + uds_enqueue_request(request, STAGE_INDEX); +} + +static int finish_previous_chapter(struct uds_index *index, u64 current_chapter_number) +{ + int result; + struct chapter_writer *writer = index->chapter_writer; + + mutex_lock(&writer->mutex); + while (index->newest_virtual_chapter < current_chapter_number) + uds_wait_cond(&writer->cond, &writer->mutex); + result = writer->result; + mutex_unlock(&writer->mutex); + + if (result != UDS_SUCCESS) + return vdo_log_error_strerror(result, + "Writing of previous open chapter failed"); + + return UDS_SUCCESS; +} + +static int swap_open_chapter(struct index_zone *zone) +{ + int result; + struct open_chapter_zone *temporary_chapter; + + result = finish_previous_chapter(zone->index, zone->newest_virtual_chapter); + if (result != UDS_SUCCESS) + return result; + + temporary_chapter = zone->open_chapter; + zone->open_chapter = zone->writing_chapter; + zone->writing_chapter = temporary_chapter; + return UDS_SUCCESS; +} + +/* + * Inform the chapter writer that this zone is done with this chapter. The chapter won't start + * writing until all zones have closed it. + */ +static unsigned int start_closing_chapter(struct uds_index *index, + unsigned int zone_number, + struct open_chapter_zone *chapter) +{ + unsigned int finished_zones; + struct chapter_writer *writer = index->chapter_writer; + + mutex_lock(&writer->mutex); + finished_zones = ++writer->zones_to_write; + writer->chapters[zone_number] = chapter; + uds_broadcast_cond(&writer->cond); + mutex_unlock(&writer->mutex); + + return finished_zones; +} + +static int announce_chapter_closed(struct index_zone *zone, u64 closed_chapter) +{ + int result; + unsigned int i; + struct uds_zone_message zone_message = { + .type = UDS_MESSAGE_ANNOUNCE_CHAPTER_CLOSED, + .virtual_chapter = closed_chapter, + }; + + for (i = 0; i < zone->index->zone_count; i++) { + if (zone->id == i) + continue; + + result = launch_zone_message(zone_message, i, zone->index); + if (result != UDS_SUCCESS) + return result; + } + + return UDS_SUCCESS; +} + +static int open_next_chapter(struct index_zone *zone) +{ + int result; + u64 closed_chapter; + u64 expiring; + unsigned int finished_zones; + u32 expire_chapters; + + vdo_log_debug("closing chapter %llu of zone %u after %u entries (%u short)", + (unsigned long long) zone->newest_virtual_chapter, zone->id, + zone->open_chapter->size, + zone->open_chapter->capacity - zone->open_chapter->size); + + result = swap_open_chapter(zone); + if (result != UDS_SUCCESS) + return result; + + closed_chapter = zone->newest_virtual_chapter++; + uds_set_volume_index_zone_open_chapter(zone->index->volume_index, zone->id, + zone->newest_virtual_chapter); + uds_reset_open_chapter(zone->open_chapter); + + finished_zones = start_closing_chapter(zone->index, zone->id, + zone->writing_chapter); + if ((finished_zones == 1) && (zone->index->zone_count > 1)) { + result = announce_chapter_closed(zone, closed_chapter); + if (result != UDS_SUCCESS) + return result; + } + + expiring = zone->oldest_virtual_chapter; + expire_chapters = uds_chapters_to_expire(zone->index->volume->geometry, + zone->newest_virtual_chapter); + zone->oldest_virtual_chapter += expire_chapters; + + if (finished_zones < zone->index->zone_count) + return UDS_SUCCESS; + + while (expire_chapters-- > 0) + uds_forget_chapter(zone->index->volume, expiring++); + + return UDS_SUCCESS; +} + +static int handle_chapter_closed(struct index_zone *zone, u64 virtual_chapter) +{ + if (zone->newest_virtual_chapter == virtual_chapter) + return open_next_chapter(zone); + + return UDS_SUCCESS; +} + +static int dispatch_index_zone_control_request(struct uds_request *request) +{ + struct uds_zone_message *message = &request->zone_message; + struct index_zone *zone = request->index->zones[request->zone_number]; + + switch (message->type) { + case UDS_MESSAGE_SPARSE_CACHE_BARRIER: + return uds_update_sparse_cache(zone, message->virtual_chapter); + + case UDS_MESSAGE_ANNOUNCE_CHAPTER_CLOSED: + return handle_chapter_closed(zone, message->virtual_chapter); + + default: + vdo_log_error("invalid message type: %d", message->type); + return UDS_INVALID_ARGUMENT; + } +} + +static void set_request_location(struct uds_request *request, + enum uds_index_region new_location) +{ + request->location = new_location; + request->found = ((new_location == UDS_LOCATION_IN_OPEN_CHAPTER) || + (new_location == UDS_LOCATION_IN_DENSE) || + (new_location == UDS_LOCATION_IN_SPARSE)); +} + +static void set_chapter_location(struct uds_request *request, + const struct index_zone *zone, u64 virtual_chapter) +{ + request->found = true; + if (virtual_chapter == zone->newest_virtual_chapter) + request->location = UDS_LOCATION_IN_OPEN_CHAPTER; + else if (is_zone_chapter_sparse(zone, virtual_chapter)) + request->location = UDS_LOCATION_IN_SPARSE; + else + request->location = UDS_LOCATION_IN_DENSE; +} + +static int search_sparse_cache_in_zone(struct index_zone *zone, struct uds_request *request, + u64 virtual_chapter, bool *found) +{ + int result; + struct volume *volume; + u16 record_page_number; + u32 chapter; + + result = uds_search_sparse_cache(zone, &request->record_name, &virtual_chapter, + &record_page_number); + if ((result != UDS_SUCCESS) || (virtual_chapter == NO_CHAPTER)) + return result; + + request->virtual_chapter = virtual_chapter; + volume = zone->index->volume; + chapter = uds_map_to_physical_chapter(volume->geometry, virtual_chapter); + return uds_search_cached_record_page(volume, request, chapter, + record_page_number, found); +} + +static int get_record_from_zone(struct index_zone *zone, struct uds_request *request, + bool *found) +{ + struct volume *volume; + + if (request->location == UDS_LOCATION_RECORD_PAGE_LOOKUP) { + *found = true; + return UDS_SUCCESS; + } else if (request->location == UDS_LOCATION_UNAVAILABLE) { + *found = false; + return UDS_SUCCESS; + } + + if (request->virtual_chapter == zone->newest_virtual_chapter) { + uds_search_open_chapter(zone->open_chapter, &request->record_name, + &request->old_metadata, found); + return UDS_SUCCESS; + } + + if ((zone->newest_virtual_chapter > 0) && + (request->virtual_chapter == (zone->newest_virtual_chapter - 1)) && + (zone->writing_chapter->size > 0)) { + uds_search_open_chapter(zone->writing_chapter, &request->record_name, + &request->old_metadata, found); + return UDS_SUCCESS; + } + + volume = zone->index->volume; + if (is_zone_chapter_sparse(zone, request->virtual_chapter) && + uds_sparse_cache_contains(volume->sparse_cache, request->virtual_chapter, + request->zone_number)) + return search_sparse_cache_in_zone(zone, request, + request->virtual_chapter, found); + + return uds_search_volume_page_cache(volume, request, found); +} + +static int put_record_in_zone(struct index_zone *zone, struct uds_request *request, + const struct uds_record_data *metadata) +{ + unsigned int remaining; + + remaining = uds_put_open_chapter(zone->open_chapter, &request->record_name, + metadata); + if (remaining == 0) + return open_next_chapter(zone); + + return UDS_SUCCESS; +} + +static int search_index_zone(struct index_zone *zone, struct uds_request *request) +{ + int result; + struct volume_index_record record; + bool overflow_record, found = false; + struct uds_record_data *metadata; + u64 chapter; + + result = uds_get_volume_index_record(zone->index->volume_index, + &request->record_name, &record); + if (result != UDS_SUCCESS) + return result; + + if (record.is_found) { + if (request->requeued && request->virtual_chapter != record.virtual_chapter) + set_request_location(request, UDS_LOCATION_UNKNOWN); + + request->virtual_chapter = record.virtual_chapter; + result = get_record_from_zone(zone, request, &found); + if (result != UDS_SUCCESS) + return result; + } + + if (found) + set_chapter_location(request, zone, record.virtual_chapter); + + /* + * If a record has overflowed a chapter index in more than one chapter (or overflowed in + * one chapter and collided with an existing record), it will exist as a collision record + * in the volume index, but we won't find it in the volume. This case needs special + * handling. + */ + overflow_record = (record.is_found && record.is_collision && !found); + chapter = zone->newest_virtual_chapter; + if (found || overflow_record) { + if ((request->type == UDS_QUERY_NO_UPDATE) || + ((request->type == UDS_QUERY) && overflow_record)) { + /* There is nothing left to do. */ + return UDS_SUCCESS; + } + + if (record.virtual_chapter != chapter) { + /* + * Update the volume index to reference the new chapter for the block. If + * the record had been deleted or dropped from the chapter index, it will + * be back. + */ + result = uds_set_volume_index_record_chapter(&record, chapter); + } else if (request->type != UDS_UPDATE) { + /* The record is already in the open chapter. */ + return UDS_SUCCESS; + } + } else { + /* + * The record wasn't in the volume index, so check whether the + * name is in a cached sparse chapter. If we found the name on + * a previous search, use that result instead. + */ + if (request->location == UDS_LOCATION_RECORD_PAGE_LOOKUP) { + found = true; + } else if (request->location == UDS_LOCATION_UNAVAILABLE) { + found = false; + } else if (uds_is_sparse_index_geometry(zone->index->volume->geometry) && + !uds_is_volume_index_sample(zone->index->volume_index, + &request->record_name)) { + result = search_sparse_cache_in_zone(zone, request, NO_CHAPTER, + &found); + if (result != UDS_SUCCESS) + return result; + } + + if (found) + set_request_location(request, UDS_LOCATION_IN_SPARSE); + + if ((request->type == UDS_QUERY_NO_UPDATE) || + ((request->type == UDS_QUERY) && !found)) { + /* There is nothing left to do. */ + return UDS_SUCCESS; + } + + /* + * Add a new entry to the volume index referencing the open chapter. This needs to + * be done both for new records, and for records from cached sparse chapters. + */ + result = uds_put_volume_index_record(&record, chapter); + } + + if (result == UDS_OVERFLOW) { + /* + * The volume index encountered a delta list overflow. The condition was already + * logged. We will go on without adding the record to the open chapter. + */ + return UDS_SUCCESS; + } + + if (result != UDS_SUCCESS) + return result; + + if (!found || (request->type == UDS_UPDATE)) { + /* This is a new record or we're updating an existing record. */ + metadata = &request->new_metadata; + } else { + /* Move the existing record to the open chapter. */ + metadata = &request->old_metadata; + } + + return put_record_in_zone(zone, request, metadata); +} + +static int remove_from_index_zone(struct index_zone *zone, struct uds_request *request) +{ + int result; + struct volume_index_record record; + + result = uds_get_volume_index_record(zone->index->volume_index, + &request->record_name, &record); + if (result != UDS_SUCCESS) + return result; + + if (!record.is_found) + return UDS_SUCCESS; + + /* If the request was requeued, check whether the saved state is still valid. */ + + if (record.is_collision) { + set_chapter_location(request, zone, record.virtual_chapter); + } else { + /* Non-collision records are hints, so resolve the name in the chapter. */ + bool found; + + if (request->requeued && request->virtual_chapter != record.virtual_chapter) + set_request_location(request, UDS_LOCATION_UNKNOWN); + + request->virtual_chapter = record.virtual_chapter; + result = get_record_from_zone(zone, request, &found); + if (result != UDS_SUCCESS) + return result; + + if (!found) { + /* There is no record to remove. */ + return UDS_SUCCESS; + } + } + + set_chapter_location(request, zone, record.virtual_chapter); + + /* + * Delete the volume index entry for the named record only. Note that a later search might + * later return stale advice if there is a colliding name in the same chapter, but it's a + * very rare case (1 in 2^21). + */ + result = uds_remove_volume_index_record(&record); + if (result != UDS_SUCCESS) + return result; + + /* + * If the record is in the open chapter, we must remove it or mark it deleted to avoid + * trouble if the record is added again later. + */ + if (request->location == UDS_LOCATION_IN_OPEN_CHAPTER) + uds_remove_from_open_chapter(zone->open_chapter, &request->record_name); + + return UDS_SUCCESS; +} + +static int dispatch_index_request(struct uds_index *index, struct uds_request *request) +{ + int result; + struct index_zone *zone = index->zones[request->zone_number]; + + if (!request->requeued) { + result = simulate_index_zone_barrier_message(zone, request); + if (result != UDS_SUCCESS) + return result; + } + + switch (request->type) { + case UDS_POST: + case UDS_UPDATE: + case UDS_QUERY: + case UDS_QUERY_NO_UPDATE: + result = search_index_zone(zone, request); + break; + + case UDS_DELETE: + result = remove_from_index_zone(zone, request); + break; + + default: + result = vdo_log_warning_strerror(UDS_INVALID_ARGUMENT, + "invalid request type: %d", + request->type); + break; + } + + return result; +} + +/* This is the request processing function invoked by each zone's thread. */ +static void execute_zone_request(struct uds_request *request) +{ + int result; + struct uds_index *index = request->index; + + if (request->zone_message.type != UDS_MESSAGE_NONE) { + result = dispatch_index_zone_control_request(request); + if (result != UDS_SUCCESS) { + vdo_log_error_strerror(result, "error executing message: %d", + request->zone_message.type); + } + + /* Once the message is processed it can be freed. */ + vdo_free(vdo_forget(request)); + return; + } + + index->need_to_save = true; + if (request->requeued && (request->status != UDS_SUCCESS)) { + set_request_location(request, UDS_LOCATION_UNAVAILABLE); + index->callback(request); + return; + } + + result = dispatch_index_request(index, request); + if (result == UDS_QUEUED) { + /* The request has been requeued so don't let it complete. */ + return; + } + + if (!request->found) + set_request_location(request, UDS_LOCATION_UNAVAILABLE); + + request->status = result; + index->callback(request); +} + +static int initialize_index_queues(struct uds_index *index, + const struct index_geometry *geometry) +{ + int result; + unsigned int i; + + for (i = 0; i < index->zone_count; i++) { + result = uds_make_request_queue("indexW", &execute_zone_request, + &index->zone_queues[i]); + if (result != UDS_SUCCESS) + return result; + } + + /* The triage queue is only needed for sparse multi-zone indexes. */ + if ((index->zone_count > 1) && uds_is_sparse_index_geometry(geometry)) { + result = uds_make_request_queue("triageW", &triage_request, + &index->triage_queue); + if (result != UDS_SUCCESS) + return result; + } + + return UDS_SUCCESS; +} + +/* This is the driver function for the chapter writer thread. */ +static void close_chapters(void *arg) +{ + int result; + struct chapter_writer *writer = arg; + struct uds_index *index = writer->index; + + vdo_log_debug("chapter writer starting"); + mutex_lock(&writer->mutex); + for (;;) { + while (writer->zones_to_write < index->zone_count) { + if (writer->stop && (writer->zones_to_write == 0)) { + /* + * We've been told to stop, and all of the zones are in the same + * open chapter, so we can exit now. + */ + mutex_unlock(&writer->mutex); + vdo_log_debug("chapter writer stopping"); + return; + } + uds_wait_cond(&writer->cond, &writer->mutex); + } + + /* + * Release the lock while closing a chapter. We probably don't need to do this, but + * it seems safer in principle. It's OK to access the chapter and chapter_number + * fields without the lock since those aren't allowed to change until we're done. + */ + mutex_unlock(&writer->mutex); + + if (index->has_saved_open_chapter) { + /* + * Remove the saved open chapter the first time we close an open chapter + * after loading from a clean shutdown, or after doing a clean save. The + * lack of the saved open chapter will indicate that a recovery is + * necessary. + */ + index->has_saved_open_chapter = false; + result = uds_discard_open_chapter(index->layout); + if (result == UDS_SUCCESS) + vdo_log_debug("Discarding saved open chapter"); + } + + result = uds_close_open_chapter(writer->chapters, index->zone_count, + index->volume, + writer->open_chapter_index, + writer->collated_records, + index->newest_virtual_chapter); + + mutex_lock(&writer->mutex); + index->newest_virtual_chapter++; + index->oldest_virtual_chapter += + uds_chapters_to_expire(index->volume->geometry, + index->newest_virtual_chapter); + writer->result = result; + writer->zones_to_write = 0; + uds_broadcast_cond(&writer->cond); + } +} + +static void stop_chapter_writer(struct chapter_writer *writer) +{ + struct thread *writer_thread = NULL; + + mutex_lock(&writer->mutex); + if (writer->thread != NULL) { + writer_thread = writer->thread; + writer->thread = NULL; + writer->stop = true; + uds_broadcast_cond(&writer->cond); + } + mutex_unlock(&writer->mutex); + + if (writer_thread != NULL) + vdo_join_threads(writer_thread); +} + +static void free_chapter_writer(struct chapter_writer *writer) +{ + if (writer == NULL) + return; + + stop_chapter_writer(writer); + uds_free_open_chapter_index(writer->open_chapter_index); + vdo_free(writer->collated_records); + vdo_free(writer); +} + +static int make_chapter_writer(struct uds_index *index, + struct chapter_writer **writer_ptr) +{ + int result; + struct chapter_writer *writer; + size_t collated_records_size = + (sizeof(struct uds_volume_record) * index->volume->geometry->records_per_chapter); + + result = vdo_allocate_extended(struct chapter_writer, index->zone_count, + struct open_chapter_zone *, "Chapter Writer", + &writer); + if (result != VDO_SUCCESS) + return result; + + writer->index = index; + mutex_init(&writer->mutex); + uds_init_cond(&writer->cond); + + result = vdo_allocate_cache_aligned(collated_records_size, "collated records", + &writer->collated_records); + if (result != VDO_SUCCESS) { + free_chapter_writer(writer); + return result; + } + + result = uds_make_open_chapter_index(&writer->open_chapter_index, + index->volume->geometry, + index->volume->nonce); + if (result != UDS_SUCCESS) { + free_chapter_writer(writer); + return result; + } + + writer->memory_size = (sizeof(struct chapter_writer) + + index->zone_count * sizeof(struct open_chapter_zone *) + + collated_records_size + + writer->open_chapter_index->memory_size); + + result = vdo_create_thread(close_chapters, writer, "writer", &writer->thread); + if (result != VDO_SUCCESS) { + free_chapter_writer(writer); + return result; + } + + *writer_ptr = writer; + return UDS_SUCCESS; +} + +static int load_index(struct uds_index *index) +{ + int result; + u64 last_save_chapter; + + result = uds_load_index_state(index->layout, index); + if (result != UDS_SUCCESS) + return UDS_INDEX_NOT_SAVED_CLEANLY; + + last_save_chapter = ((index->last_save != NO_LAST_SAVE) ? index->last_save : 0); + + vdo_log_info("loaded index from chapter %llu through chapter %llu", + (unsigned long long) index->oldest_virtual_chapter, + (unsigned long long) last_save_chapter); + + return UDS_SUCCESS; +} + +static int rebuild_index_page_map(struct uds_index *index, u64 vcn) +{ + int result; + struct delta_index_page *chapter_index_page; + struct index_geometry *geometry = index->volume->geometry; + u32 chapter = uds_map_to_physical_chapter(geometry, vcn); + u32 expected_list_number = 0; + u32 index_page_number; + u32 lowest_delta_list; + u32 highest_delta_list; + + for (index_page_number = 0; + index_page_number < geometry->index_pages_per_chapter; + index_page_number++) { + result = uds_get_volume_index_page(index->volume, chapter, + index_page_number, + &chapter_index_page); + if (result != UDS_SUCCESS) { + return vdo_log_error_strerror(result, + "failed to read index page %u in chapter %u", + index_page_number, chapter); + } + + lowest_delta_list = chapter_index_page->lowest_list_number; + highest_delta_list = chapter_index_page->highest_list_number; + if (lowest_delta_list != expected_list_number) { + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "chapter %u index page %u is corrupt", + chapter, index_page_number); + } + + uds_update_index_page_map(index->volume->index_page_map, vcn, chapter, + index_page_number, highest_delta_list); + expected_list_number = highest_delta_list + 1; + } + + return UDS_SUCCESS; +} + +static int replay_record(struct uds_index *index, const struct uds_record_name *name, + u64 virtual_chapter, bool will_be_sparse_chapter) +{ + int result; + struct volume_index_record record; + bool update_record; + + if (will_be_sparse_chapter && + !uds_is_volume_index_sample(index->volume_index, name)) { + /* + * This entry will be in a sparse chapter after the rebuild completes, and it is + * not a sample, so just skip over it. + */ + return UDS_SUCCESS; + } + + result = uds_get_volume_index_record(index->volume_index, name, &record); + if (result != UDS_SUCCESS) + return result; + + if (record.is_found) { + if (record.is_collision) { + if (record.virtual_chapter == virtual_chapter) { + /* The record is already correct. */ + return UDS_SUCCESS; + } + + update_record = true; + } else if (record.virtual_chapter == virtual_chapter) { + /* + * There is a volume index entry pointing to the current chapter, but we + * don't know if it is for the same name as the one we are currently + * working on or not. For now, we're just going to assume that it isn't. + * This will create one extra collision record if there was a deleted + * record in the current chapter. + */ + update_record = false; + } else { + /* + * If we're rebuilding, we don't normally want to go to disk to see if the + * record exists, since we will likely have just read the record from disk + * (i.e. we know it's there). The exception to this is when we find an + * entry in the volume index that has a different chapter. In this case, we + * need to search that chapter to determine if the volume index entry was + * for the same record or a different one. + */ + result = uds_search_volume_page_cache_for_rebuild(index->volume, + name, + record.virtual_chapter, + &update_record); + if (result != UDS_SUCCESS) + return result; + } + } else { + update_record = false; + } + + if (update_record) { + /* + * Update the volume index to reference the new chapter for the block. If the + * record had been deleted or dropped from the chapter index, it will be back. + */ + result = uds_set_volume_index_record_chapter(&record, virtual_chapter); + } else { + /* + * Add a new entry to the volume index referencing the open chapter. This should be + * done regardless of whether we are a brand new record or a sparse record, i.e. + * one that doesn't exist in the index but does on disk, since for a sparse record, + * we would want to un-sparsify if it did exist. + */ + result = uds_put_volume_index_record(&record, virtual_chapter); + } + + if ((result == UDS_DUPLICATE_NAME) || (result == UDS_OVERFLOW)) { + /* The rebuilt index will lose these records. */ + return UDS_SUCCESS; + } + + return result; +} + +static bool check_for_suspend(struct uds_index *index) +{ + bool closing; + + if (index->load_context == NULL) + return false; + + mutex_lock(&index->load_context->mutex); + if (index->load_context->status != INDEX_SUSPENDING) { + mutex_unlock(&index->load_context->mutex); + return false; + } + + /* Notify that we are suspended and wait for the resume. */ + index->load_context->status = INDEX_SUSPENDED; + uds_broadcast_cond(&index->load_context->cond); + + while ((index->load_context->status != INDEX_OPENING) && + (index->load_context->status != INDEX_FREEING)) + uds_wait_cond(&index->load_context->cond, &index->load_context->mutex); + + closing = (index->load_context->status == INDEX_FREEING); + mutex_unlock(&index->load_context->mutex); + return closing; +} + +static int replay_chapter(struct uds_index *index, u64 virtual, bool sparse) +{ + int result; + u32 i; + u32 j; + const struct index_geometry *geometry; + u32 physical_chapter; + + if (check_for_suspend(index)) { + vdo_log_info("Replay interrupted by index shutdown at chapter %llu", + (unsigned long long) virtual); + return -EBUSY; + } + + geometry = index->volume->geometry; + physical_chapter = uds_map_to_physical_chapter(geometry, virtual); + uds_prefetch_volume_chapter(index->volume, physical_chapter); + uds_set_volume_index_open_chapter(index->volume_index, virtual); + + result = rebuild_index_page_map(index, virtual); + if (result != UDS_SUCCESS) { + return vdo_log_error_strerror(result, + "could not rebuild index page map for chapter %u", + physical_chapter); + } + + for (i = 0; i < geometry->record_pages_per_chapter; i++) { + u8 *record_page; + u32 record_page_number; + + record_page_number = geometry->index_pages_per_chapter + i; + result = uds_get_volume_record_page(index->volume, physical_chapter, + record_page_number, &record_page); + if (result != UDS_SUCCESS) { + return vdo_log_error_strerror(result, "could not get page %d", + record_page_number); + } + + for (j = 0; j < geometry->records_per_page; j++) { + const u8 *name_bytes; + struct uds_record_name name; + + name_bytes = record_page + (j * BYTES_PER_RECORD); + memcpy(&name.name, name_bytes, UDS_RECORD_NAME_SIZE); + result = replay_record(index, &name, virtual, sparse); + if (result != UDS_SUCCESS) + return result; + } + } + + return UDS_SUCCESS; +} + +static int replay_volume(struct uds_index *index) +{ + int result; + u64 old_map_update; + u64 new_map_update; + u64 virtual; + u64 from_virtual = index->oldest_virtual_chapter; + u64 upto_virtual = index->newest_virtual_chapter; + bool will_be_sparse; + + vdo_log_info("Replaying volume from chapter %llu through chapter %llu", + (unsigned long long) from_virtual, + (unsigned long long) upto_virtual); + + /* + * The index failed to load, so the volume index is empty. Add records to the volume index + * in order, skipping non-hooks in chapters which will be sparse to save time. + * + * Go through each record page of each chapter and add the records back to the volume + * index. This should not cause anything to be written to either the open chapter or the + * on-disk volume. Also skip the on-disk chapter corresponding to upto_virtual, as this + * would have already been purged from the volume index when the chapter was opened. + * + * Also, go through each index page for each chapter and rebuild the index page map. + */ + old_map_update = index->volume->index_page_map->last_update; + for (virtual = from_virtual; virtual < upto_virtual; virtual++) { + will_be_sparse = uds_is_chapter_sparse(index->volume->geometry, + from_virtual, upto_virtual, + virtual); + result = replay_chapter(index, virtual, will_be_sparse); + if (result != UDS_SUCCESS) + return result; + } + + /* Also reap the chapter being replaced by the open chapter. */ + uds_set_volume_index_open_chapter(index->volume_index, upto_virtual); + + new_map_update = index->volume->index_page_map->last_update; + if (new_map_update != old_map_update) { + vdo_log_info("replay changed index page map update from %llu to %llu", + (unsigned long long) old_map_update, + (unsigned long long) new_map_update); + } + + return UDS_SUCCESS; +} + +static int rebuild_index(struct uds_index *index) +{ + int result; + u64 lowest; + u64 highest; + bool is_empty = false; + u32 chapters_per_volume = index->volume->geometry->chapters_per_volume; + + index->volume->lookup_mode = LOOKUP_FOR_REBUILD; + result = uds_find_volume_chapter_boundaries(index->volume, &lowest, &highest, + &is_empty); + if (result != UDS_SUCCESS) { + return vdo_log_fatal_strerror(result, + "cannot rebuild index: unknown volume chapter boundaries"); + } + + if (is_empty) { + index->newest_virtual_chapter = 0; + index->oldest_virtual_chapter = 0; + index->volume->lookup_mode = LOOKUP_NORMAL; + return UDS_SUCCESS; + } + + index->newest_virtual_chapter = highest + 1; + index->oldest_virtual_chapter = lowest; + if (index->newest_virtual_chapter == + (index->oldest_virtual_chapter + chapters_per_volume)) { + /* Skip the chapter shadowed by the open chapter. */ + index->oldest_virtual_chapter++; + } + + result = replay_volume(index); + if (result != UDS_SUCCESS) + return result; + + index->volume->lookup_mode = LOOKUP_NORMAL; + return UDS_SUCCESS; +} + +static void free_index_zone(struct index_zone *zone) +{ + if (zone == NULL) + return; + + uds_free_open_chapter(zone->open_chapter); + uds_free_open_chapter(zone->writing_chapter); + vdo_free(zone); +} + +static int make_index_zone(struct uds_index *index, unsigned int zone_number) +{ + int result; + struct index_zone *zone; + + result = vdo_allocate(1, struct index_zone, "index zone", &zone); + if (result != VDO_SUCCESS) + return result; + + result = uds_make_open_chapter(index->volume->geometry, index->zone_count, + &zone->open_chapter); + if (result != UDS_SUCCESS) { + free_index_zone(zone); + return result; + } + + result = uds_make_open_chapter(index->volume->geometry, index->zone_count, + &zone->writing_chapter); + if (result != UDS_SUCCESS) { + free_index_zone(zone); + return result; + } + + zone->index = index; + zone->id = zone_number; + index->zones[zone_number] = zone; + + return UDS_SUCCESS; +} + +int uds_make_index(struct uds_configuration *config, enum uds_open_index_type open_type, + struct index_load_context *load_context, index_callback_fn callback, + struct uds_index **new_index) +{ + int result; + bool loaded = false; + bool new = (open_type == UDS_CREATE); + struct uds_index *index = NULL; + struct index_zone *zone; + u64 nonce; + unsigned int z; + + result = vdo_allocate_extended(struct uds_index, config->zone_count, + struct uds_request_queue *, "index", &index); + if (result != VDO_SUCCESS) + return result; + + index->zone_count = config->zone_count; + + result = uds_make_index_layout(config, new, &index->layout); + if (result != UDS_SUCCESS) { + uds_free_index(index); + return result; + } + + result = vdo_allocate(index->zone_count, struct index_zone *, "zones", + &index->zones); + if (result != VDO_SUCCESS) { + uds_free_index(index); + return result; + } + + result = uds_make_volume(config, index->layout, &index->volume); + if (result != UDS_SUCCESS) { + uds_free_index(index); + return result; + } + + index->volume->lookup_mode = LOOKUP_NORMAL; + for (z = 0; z < index->zone_count; z++) { + result = make_index_zone(index, z); + if (result != UDS_SUCCESS) { + uds_free_index(index); + return vdo_log_error_strerror(result, + "Could not create index zone"); + } + } + + nonce = uds_get_volume_nonce(index->layout); + result = uds_make_volume_index(config, nonce, &index->volume_index); + if (result != UDS_SUCCESS) { + uds_free_index(index); + return vdo_log_error_strerror(result, "could not make volume index"); + } + + index->load_context = load_context; + index->callback = callback; + + result = initialize_index_queues(index, config->geometry); + if (result != UDS_SUCCESS) { + uds_free_index(index); + return result; + } + + result = make_chapter_writer(index, &index->chapter_writer); + if (result != UDS_SUCCESS) { + uds_free_index(index); + return result; + } + + if (!new) { + result = load_index(index); + switch (result) { + case UDS_SUCCESS: + loaded = true; + break; + case -ENOMEM: + /* We should not try a rebuild for this error. */ + vdo_log_error_strerror(result, "index could not be loaded"); + break; + default: + vdo_log_error_strerror(result, "index could not be loaded"); + if (open_type == UDS_LOAD) { + result = rebuild_index(index); + if (result != UDS_SUCCESS) { + vdo_log_error_strerror(result, + "index could not be rebuilt"); + } + } + break; + } + } + + if (result != UDS_SUCCESS) { + uds_free_index(index); + return vdo_log_error_strerror(result, "fatal error in %s()", __func__); + } + + for (z = 0; z < index->zone_count; z++) { + zone = index->zones[z]; + zone->oldest_virtual_chapter = index->oldest_virtual_chapter; + zone->newest_virtual_chapter = index->newest_virtual_chapter; + } + + if (index->load_context != NULL) { + mutex_lock(&index->load_context->mutex); + index->load_context->status = INDEX_READY; + /* + * If we get here, suspend is meaningless, but notify any thread trying to suspend + * us so it doesn't hang. + */ + uds_broadcast_cond(&index->load_context->cond); + mutex_unlock(&index->load_context->mutex); + } + + index->has_saved_open_chapter = loaded; + index->need_to_save = !loaded; + *new_index = index; + return UDS_SUCCESS; +} + +void uds_free_index(struct uds_index *index) +{ + unsigned int i; + + if (index == NULL) + return; + + uds_request_queue_finish(index->triage_queue); + for (i = 0; i < index->zone_count; i++) + uds_request_queue_finish(index->zone_queues[i]); + + free_chapter_writer(index->chapter_writer); + + uds_free_volume_index(index->volume_index); + if (index->zones != NULL) { + for (i = 0; i < index->zone_count; i++) + free_index_zone(index->zones[i]); + vdo_free(index->zones); + } + + uds_free_volume(index->volume); + uds_free_index_layout(vdo_forget(index->layout)); + vdo_free(index); +} + +/* Wait for the chapter writer to complete any outstanding writes. */ +void uds_wait_for_idle_index(struct uds_index *index) +{ + struct chapter_writer *writer = index->chapter_writer; + + mutex_lock(&writer->mutex); + while (writer->zones_to_write > 0) + uds_wait_cond(&writer->cond, &writer->mutex); + mutex_unlock(&writer->mutex); +} + +/* This function assumes that all requests have been drained. */ +int uds_save_index(struct uds_index *index) +{ + int result; + + if (!index->need_to_save) + return UDS_SUCCESS; + + uds_wait_for_idle_index(index); + index->prev_save = index->last_save; + index->last_save = ((index->newest_virtual_chapter == 0) ? + NO_LAST_SAVE : index->newest_virtual_chapter - 1); + vdo_log_info("beginning save (vcn %llu)", (unsigned long long) index->last_save); + + result = uds_save_index_state(index->layout, index); + if (result != UDS_SUCCESS) { + vdo_log_info("save index failed"); + index->last_save = index->prev_save; + } else { + index->has_saved_open_chapter = true; + index->need_to_save = false; + vdo_log_info("finished save (vcn %llu)", + (unsigned long long) index->last_save); + } + + return result; +} + +int uds_replace_index_storage(struct uds_index *index, struct block_device *bdev) +{ + return uds_replace_volume_storage(index->volume, index->layout, bdev); +} + +/* Accessing statistics should be safe from any thread. */ +void uds_get_index_stats(struct uds_index *index, struct uds_index_stats *counters) +{ + struct volume_index_stats stats; + + uds_get_volume_index_stats(index->volume_index, &stats); + counters->entries_indexed = stats.record_count; + counters->collisions = stats.collision_count; + counters->entries_discarded = stats.discard_count; + + counters->memory_used = (index->volume_index->memory_size + + index->volume->cache_size + + index->chapter_writer->memory_size); +} + +void uds_enqueue_request(struct uds_request *request, enum request_stage stage) +{ + struct uds_index *index = request->index; + struct uds_request_queue *queue; + + switch (stage) { + case STAGE_TRIAGE: + if (index->triage_queue != NULL) { + queue = index->triage_queue; + break; + } + + fallthrough; + + case STAGE_INDEX: + request->zone_number = + uds_get_volume_index_zone(index->volume_index, &request->record_name); + fallthrough; + + case STAGE_MESSAGE: + queue = index->zone_queues[request->zone_number]; + break; + + default: + VDO_ASSERT_LOG_ONLY(false, "invalid index stage: %d", stage); + return; + } + + uds_request_queue_enqueue(queue, request); +} diff --git a/drivers/md/dm-vdo/indexer/index.h b/drivers/md/dm-vdo/indexer/index.h new file mode 100644 index 000000000000..edabb239548e --- /dev/null +++ b/drivers/md/dm-vdo/indexer/index.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_INDEX_H +#define UDS_INDEX_H + +#include "index-layout.h" +#include "index-session.h" +#include "open-chapter.h" +#include "volume.h" +#include "volume-index.h" + +/* + * The index is a high-level structure which represents the totality of the UDS index. It manages + * the queues for incoming requests and dispatches them to the appropriate sub-components like the + * volume or the volume index. It also manages administrative tasks such as saving and loading the + * index. + * + * The index is divided into a number of independent zones and assigns each request to a zone based + * on its name. Most sub-components are similarly divided into zones as well so that requests in + * each zone usually operate without interference or coordination between zones. + */ + +typedef void (*index_callback_fn)(struct uds_request *request); + +struct index_zone { + struct uds_index *index; + struct open_chapter_zone *open_chapter; + struct open_chapter_zone *writing_chapter; + u64 oldest_virtual_chapter; + u64 newest_virtual_chapter; + unsigned int id; +}; + +struct uds_index { + bool has_saved_open_chapter; + bool need_to_save; + struct index_load_context *load_context; + struct index_layout *layout; + struct volume_index *volume_index; + struct volume *volume; + unsigned int zone_count; + struct index_zone **zones; + + u64 oldest_virtual_chapter; + u64 newest_virtual_chapter; + + u64 last_save; + u64 prev_save; + struct chapter_writer *chapter_writer; + + index_callback_fn callback; + struct uds_request_queue *triage_queue; + struct uds_request_queue *zone_queues[]; +}; + +enum request_stage { + STAGE_TRIAGE, + STAGE_INDEX, + STAGE_MESSAGE, +}; + +int __must_check uds_make_index(struct uds_configuration *config, + enum uds_open_index_type open_type, + struct index_load_context *load_context, + index_callback_fn callback, struct uds_index **new_index); + +int __must_check uds_save_index(struct uds_index *index); + +void uds_free_index(struct uds_index *index); + +int __must_check uds_replace_index_storage(struct uds_index *index, + struct block_device *bdev); + +void uds_get_index_stats(struct uds_index *index, struct uds_index_stats *counters); + +void uds_enqueue_request(struct uds_request *request, enum request_stage stage); + +void uds_wait_for_idle_index(struct uds_index *index); + +#endif /* UDS_INDEX_H */ diff --git a/drivers/md/dm-vdo/indexer/indexer.h b/drivers/md/dm-vdo/indexer/indexer.h new file mode 100644 index 000000000000..3744aaf625b0 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/indexer.h @@ -0,0 +1,353 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef INDEXER_H +#define INDEXER_H + +#include +#include +#include +#include + +#include "funnel-queue.h" + +/* + * UDS public API + * + * The Universal Deduplication System (UDS) is an efficient name-value store. When used for + * deduplicating storage, the names are generally hashes of data blocks and the associated data is + * where that block is located on the underlying storage medium. The stored names are expected to + * be randomly distributed among the space of possible names. If this assumption is violated, the + * UDS index will store fewer names than normal but will otherwise continue to work. The data + * associated with each name can be any 16-byte value. + * + * A client must first create an index session to interact with an index. Once created, the session + * can be shared among multiple threads or users. When a session is destroyed, it will also close + * and save any associated index. + * + * To make a request, a client must allocate a uds_request structure and set the required fields + * before launching it. UDS will invoke the provided callback to complete the request. After the + * callback has been called, the uds_request structure can be freed or reused for a new request. + * There are five types of requests: + * + * A UDS_UPDATE request will associate the provided name with the provided data. Any previous data + * associated with that name will be discarded. + * + * A UDS_QUERY request will return the data associated with the provided name, if any. The entry + * for the name will also be marked as most recent, as if the data had been updated. + * + * A UDS_POST request is a combination of UDS_QUERY and UDS_UPDATE. If there is already data + * associated with the provided name, that data is returned. If there is no existing association, + * the name is associated with the newly provided data. This request is equivalent to a UDS_QUERY + * request followed by a UDS_UPDATE request if no data is found, but it is much more efficient. + * + * A UDS_QUERY_NO_UPDATE request will return the data associated with the provided name, but will + * not change the recency of the entry for the name. This request is primarily useful for testing, + * to determine whether an entry exists without changing the internal state of the index. + * + * A UDS_DELETE request removes any data associated with the provided name. This operation is + * generally not necessary, because the index will automatically discard its oldest entries once it + * becomes full. + */ + +/* General UDS constants and structures */ + +enum uds_request_type { + /* Create or update the mapping for a name, and make the name most recent. */ + UDS_UPDATE, + + /* Return any mapped data for a name, and make the name most recent. */ + UDS_QUERY, + + /* + * Return any mapped data for a name, or map the provided data to the name if there is no + * current data, and make the name most recent. + */ + UDS_POST, + + /* Return any mapped data for a name without updating its recency. */ + UDS_QUERY_NO_UPDATE, + + /* Remove any mapping for a name. */ + UDS_DELETE, + +}; + +enum uds_open_index_type { + /* Create a new index. */ + UDS_CREATE, + + /* Load an existing index and try to recover if necessary. */ + UDS_LOAD, + + /* Load an existing index, but only if it was saved cleanly. */ + UDS_NO_REBUILD, +}; + +enum { + /* The record name size in bytes */ + UDS_RECORD_NAME_SIZE = 16, + /* The maximum record data size in bytes */ + UDS_RECORD_DATA_SIZE = 16, +}; + +/* + * A type representing a UDS memory configuration which is either a positive integer number of + * gigabytes or one of the six special constants for configurations smaller than one gigabyte. + */ +typedef int uds_memory_config_size_t; + +enum { + /* The maximum configurable amount of memory */ + UDS_MEMORY_CONFIG_MAX = 1024, + /* Flag indicating that the index has one less chapter than usual */ + UDS_MEMORY_CONFIG_REDUCED = 0x1000, + UDS_MEMORY_CONFIG_REDUCED_MAX = 1024 + UDS_MEMORY_CONFIG_REDUCED, + /* Special values indicating sizes less than 1 GB */ + UDS_MEMORY_CONFIG_256MB = -256, + UDS_MEMORY_CONFIG_512MB = -512, + UDS_MEMORY_CONFIG_768MB = -768, + UDS_MEMORY_CONFIG_REDUCED_256MB = -1280, + UDS_MEMORY_CONFIG_REDUCED_512MB = -1536, + UDS_MEMORY_CONFIG_REDUCED_768MB = -1792, +}; + +struct uds_record_name { + unsigned char name[UDS_RECORD_NAME_SIZE]; +}; + +struct uds_record_data { + unsigned char data[UDS_RECORD_DATA_SIZE]; +}; + +struct uds_volume_record { + struct uds_record_name name; + struct uds_record_data data; +}; + +struct uds_parameters { + /* The block_device used for storage */ + struct block_device *bdev; + /* The maximum allowable size of the index on storage */ + size_t size; + /* The offset where the index should start */ + off_t offset; + /* The maximum memory allocation, in GB */ + uds_memory_config_size_t memory_size; + /* Whether the index should include sparse chapters */ + bool sparse; + /* A 64-bit nonce to validate the index */ + u64 nonce; + /* The number of threads used to process index requests */ + unsigned int zone_count; + /* The number of threads used to read volume pages */ + unsigned int read_threads; +}; + +/* + * These statistics capture characteristics of the current index, including resource usage and + * requests processed since the index was opened. + */ +struct uds_index_stats { + /* The total number of records stored in the index */ + u64 entries_indexed; + /* An estimate of the index's memory usage, in bytes */ + u64 memory_used; + /* The number of collisions recorded in the volume index */ + u64 collisions; + /* The number of entries discarded from the index since startup */ + u64 entries_discarded; + /* The time at which these statistics were fetched */ + s64 current_time; + /* The number of post calls that found an existing entry */ + u64 posts_found; + /* The number of post calls that added an entry */ + u64 posts_not_found; + /* + * The number of post calls that found an existing entry that is current enough to only + * exist in memory and not have been committed to disk yet + */ + u64 in_memory_posts_found; + /* + * The number of post calls that found an existing entry in the dense portion of the index + */ + u64 dense_posts_found; + /* + * The number of post calls that found an existing entry in the sparse portion of the index + */ + u64 sparse_posts_found; + /* The number of update calls that updated an existing entry */ + u64 updates_found; + /* The number of update calls that added a new entry */ + u64 updates_not_found; + /* The number of delete requests that deleted an existing entry */ + u64 deletions_found; + /* The number of delete requests that did nothing */ + u64 deletions_not_found; + /* The number of query calls that found existing entry */ + u64 queries_found; + /* The number of query calls that did not find an entry */ + u64 queries_not_found; + /* The total number of requests processed */ + u64 requests; +}; + +enum uds_index_region { + /* No location information has been determined */ + UDS_LOCATION_UNKNOWN = 0, + /* The index page entry has been found */ + UDS_LOCATION_INDEX_PAGE_LOOKUP, + /* The record page entry has been found */ + UDS_LOCATION_RECORD_PAGE_LOOKUP, + /* The record is not in the index */ + UDS_LOCATION_UNAVAILABLE, + /* The record was found in the open chapter */ + UDS_LOCATION_IN_OPEN_CHAPTER, + /* The record was found in the dense part of the index */ + UDS_LOCATION_IN_DENSE, + /* The record was found in the sparse part of the index */ + UDS_LOCATION_IN_SPARSE, +} __packed; + +/* Zone message requests are used to communicate between index zones. */ +enum uds_zone_message_type { + /* A standard request with no message */ + UDS_MESSAGE_NONE = 0, + /* Add a chapter to the sparse chapter index cache */ + UDS_MESSAGE_SPARSE_CACHE_BARRIER, + /* Close a chapter to keep the zone from falling behind */ + UDS_MESSAGE_ANNOUNCE_CHAPTER_CLOSED, +} __packed; + +struct uds_zone_message { + /* The type of message, determining how it will be processed */ + enum uds_zone_message_type type; + /* The virtual chapter number to which the message applies */ + u64 virtual_chapter; +}; + +struct uds_index_session; +struct uds_index; +struct uds_request; + +/* Once this callback has been invoked, the uds_request structure can be reused or freed. */ +typedef void (*uds_request_callback_fn)(struct uds_request *request); + +struct uds_request { + /* These input fields must be set before launching a request. */ + + /* The name of the record to look up or create */ + struct uds_record_name record_name; + /* New data to associate with the record name, if applicable */ + struct uds_record_data new_metadata; + /* A callback to invoke when the request is complete */ + uds_request_callback_fn callback; + /* The index session that will manage this request */ + struct uds_index_session *session; + /* The type of operation to perform, as describe above */ + enum uds_request_type type; + + /* These output fields are set when a request is complete. */ + + /* The existing data associated with the request name, if any */ + struct uds_record_data old_metadata; + /* Either UDS_SUCCESS or an error code for the request */ + int status; + /* True if the record name had an existing entry in the index */ + bool found; + + /* + * The remaining fields are used internally and should not be altered by clients. The index + * relies on zone_number being the first field in this section. + */ + + /* The number of the zone which will process this request*/ + unsigned int zone_number; + /* A link for adding a request to a lock-free queue */ + struct funnel_queue_entry queue_link; + /* A link for adding a request to a standard linked list */ + struct uds_request *next_request; + /* A pointer to the index processing this request */ + struct uds_index *index; + /* Control message for coordinating between zones */ + struct uds_zone_message zone_message; + /* If true, process request immediately by waking the worker thread */ + bool unbatched; + /* If true, continue this request before processing newer requests */ + bool requeued; + /* The virtual chapter containing the record name, if known */ + u64 virtual_chapter; + /* The region of the index containing the record name */ + enum uds_index_region location; +}; + +/* Compute the number of bytes needed to store an index. */ +int __must_check uds_compute_index_size(const struct uds_parameters *parameters, + u64 *index_size); + +/* A session is required for most index operations. */ +int __must_check uds_create_index_session(struct uds_index_session **session); + +/* Destroying an index session also closes and saves the associated index. */ +int uds_destroy_index_session(struct uds_index_session *session); + +/* + * Create or open an index with an existing session. This operation fails if the index session is + * suspended, or if there is already an open index. + */ +int __must_check uds_open_index(enum uds_open_index_type open_type, + const struct uds_parameters *parameters, + struct uds_index_session *session); + +/* + * Wait until all callbacks for index operations are complete, and prevent new index operations + * from starting. New index operations will fail with EBUSY until the session is resumed. Also + * optionally saves the index. + */ +int __must_check uds_suspend_index_session(struct uds_index_session *session, bool save); + +/* + * Allow new index operations for an index, whether it was suspended or not. If the index is + * suspended and the supplied block device differs from the current backing store, the index will + * start using the new backing store instead. + */ +int __must_check uds_resume_index_session(struct uds_index_session *session, + struct block_device *bdev); + +/* Wait until all outstanding index operations are complete. */ +int __must_check uds_flush_index_session(struct uds_index_session *session); + +/* Close an index. This operation fails if the index session is suspended. */ +int __must_check uds_close_index(struct uds_index_session *session); + +/* Get index statistics since the last time the index was opened. */ +int __must_check uds_get_index_session_stats(struct uds_index_session *session, + struct uds_index_stats *stats); + +/* This function will fail if any required field of the request is not set. */ +int __must_check uds_launch_request(struct uds_request *request); + +struct cond_var { + wait_queue_head_t wait_queue; +}; + +static inline void uds_init_cond(struct cond_var *cv) +{ + init_waitqueue_head(&cv->wait_queue); +} + +static inline void uds_signal_cond(struct cond_var *cv) +{ + wake_up(&cv->wait_queue); +} + +static inline void uds_broadcast_cond(struct cond_var *cv) +{ + wake_up_all(&cv->wait_queue); +} + +void uds_wait_cond(struct cond_var *cv, struct mutex *mutex); + +#endif /* INDEXER_H */ diff --git a/drivers/md/dm-vdo/indexer/io-factory.c b/drivers/md/dm-vdo/indexer/io-factory.c new file mode 100644 index 000000000000..515765d35794 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/io-factory.c @@ -0,0 +1,415 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "io-factory.h" + +#include +#include +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" + +/* + * The I/O factory object manages access to index storage, which is a contiguous range of blocks on + * a block device. + * + * The factory holds the open device and is responsible for closing it. The factory has methods to + * make helper structures that can be used to access sections of the index. + */ +struct io_factory { + struct block_device *bdev; + atomic_t ref_count; +}; + +/* The buffered reader allows efficient I/O by reading page-sized segments into a buffer. */ +struct buffered_reader { + struct io_factory *factory; + struct dm_bufio_client *client; + struct dm_buffer *buffer; + sector_t limit; + sector_t block_number; + u8 *start; + u8 *end; +}; + +#define MAX_READ_AHEAD_BLOCKS 4 + +/* + * The buffered writer allows efficient I/O by buffering writes and committing page-sized segments + * to storage. + */ +struct buffered_writer { + struct io_factory *factory; + struct dm_bufio_client *client; + struct dm_buffer *buffer; + sector_t limit; + sector_t block_number; + u8 *start; + u8 *end; + int error; +}; + +static void uds_get_io_factory(struct io_factory *factory) +{ + atomic_inc(&factory->ref_count); +} + +int uds_make_io_factory(struct block_device *bdev, struct io_factory **factory_ptr) +{ + int result; + struct io_factory *factory; + + result = vdo_allocate(1, struct io_factory, __func__, &factory); + if (result != VDO_SUCCESS) + return result; + + factory->bdev = bdev; + atomic_set_release(&factory->ref_count, 1); + + *factory_ptr = factory; + return UDS_SUCCESS; +} + +int uds_replace_storage(struct io_factory *factory, struct block_device *bdev) +{ + factory->bdev = bdev; + return UDS_SUCCESS; +} + +/* Free an I/O factory once all references have been released. */ +void uds_put_io_factory(struct io_factory *factory) +{ + if (atomic_add_return(-1, &factory->ref_count) <= 0) + vdo_free(factory); +} + +size_t uds_get_writable_size(struct io_factory *factory) +{ + return i_size_read(factory->bdev->bd_inode); +} + +/* Create a struct dm_bufio_client for an index region starting at offset. */ +int uds_make_bufio(struct io_factory *factory, off_t block_offset, size_t block_size, + unsigned int reserved_buffers, struct dm_bufio_client **client_ptr) +{ + struct dm_bufio_client *client; + + client = dm_bufio_client_create(factory->bdev, block_size, reserved_buffers, 0, + NULL, NULL, 0); + if (IS_ERR(client)) + return -PTR_ERR(client); + + dm_bufio_set_sector_offset(client, block_offset * SECTORS_PER_BLOCK); + *client_ptr = client; + return UDS_SUCCESS; +} + +static void read_ahead(struct buffered_reader *reader, sector_t block_number) +{ + if (block_number < reader->limit) { + sector_t read_ahead = min((sector_t) MAX_READ_AHEAD_BLOCKS, + reader->limit - block_number); + + dm_bufio_prefetch(reader->client, block_number, read_ahead); + } +} + +void uds_free_buffered_reader(struct buffered_reader *reader) +{ + if (reader == NULL) + return; + + if (reader->buffer != NULL) + dm_bufio_release(reader->buffer); + + dm_bufio_client_destroy(reader->client); + uds_put_io_factory(reader->factory); + vdo_free(reader); +} + +/* Create a buffered reader for an index region starting at offset. */ +int uds_make_buffered_reader(struct io_factory *factory, off_t offset, u64 block_count, + struct buffered_reader **reader_ptr) +{ + int result; + struct dm_bufio_client *client = NULL; + struct buffered_reader *reader = NULL; + + result = uds_make_bufio(factory, offset, UDS_BLOCK_SIZE, 1, &client); + if (result != UDS_SUCCESS) + return result; + + result = vdo_allocate(1, struct buffered_reader, "buffered reader", &reader); + if (result != VDO_SUCCESS) { + dm_bufio_client_destroy(client); + return result; + } + + *reader = (struct buffered_reader) { + .factory = factory, + .client = client, + .buffer = NULL, + .limit = block_count, + .block_number = 0, + .start = NULL, + .end = NULL, + }; + + read_ahead(reader, 0); + uds_get_io_factory(factory); + *reader_ptr = reader; + return UDS_SUCCESS; +} + +static int position_reader(struct buffered_reader *reader, sector_t block_number, + off_t offset) +{ + struct dm_buffer *buffer = NULL; + void *data; + + if ((reader->end == NULL) || (block_number != reader->block_number)) { + if (block_number >= reader->limit) + return UDS_OUT_OF_RANGE; + + if (reader->buffer != NULL) + dm_bufio_release(vdo_forget(reader->buffer)); + + data = dm_bufio_read(reader->client, block_number, &buffer); + if (IS_ERR(data)) + return -PTR_ERR(data); + + reader->buffer = buffer; + reader->start = data; + if (block_number == reader->block_number + 1) + read_ahead(reader, block_number + 1); + } + + reader->block_number = block_number; + reader->end = reader->start + offset; + return UDS_SUCCESS; +} + +static size_t bytes_remaining_in_read_buffer(struct buffered_reader *reader) +{ + return (reader->end == NULL) ? 0 : reader->start + UDS_BLOCK_SIZE - reader->end; +} + +static int reset_reader(struct buffered_reader *reader) +{ + sector_t block_number; + + if (bytes_remaining_in_read_buffer(reader) > 0) + return UDS_SUCCESS; + + block_number = reader->block_number; + if (reader->end != NULL) + block_number++; + + return position_reader(reader, block_number, 0); +} + +int uds_read_from_buffered_reader(struct buffered_reader *reader, u8 *data, + size_t length) +{ + int result = UDS_SUCCESS; + size_t chunk_size; + + while (length > 0) { + result = reset_reader(reader); + if (result != UDS_SUCCESS) + return result; + + chunk_size = min(length, bytes_remaining_in_read_buffer(reader)); + memcpy(data, reader->end, chunk_size); + length -= chunk_size; + data += chunk_size; + reader->end += chunk_size; + } + + return UDS_SUCCESS; +} + +/* + * Verify that the next data on the reader matches the required value. If the value matches, the + * matching contents are consumed. If the value does not match, the reader state is unchanged. + */ +int uds_verify_buffered_data(struct buffered_reader *reader, const u8 *value, + size_t length) +{ + int result = UDS_SUCCESS; + size_t chunk_size; + sector_t start_block_number = reader->block_number; + int start_offset = reader->end - reader->start; + + while (length > 0) { + result = reset_reader(reader); + if (result != UDS_SUCCESS) { + result = UDS_CORRUPT_DATA; + break; + } + + chunk_size = min(length, bytes_remaining_in_read_buffer(reader)); + if (memcmp(value, reader->end, chunk_size) != 0) { + result = UDS_CORRUPT_DATA; + break; + } + + length -= chunk_size; + value += chunk_size; + reader->end += chunk_size; + } + + if (result != UDS_SUCCESS) + position_reader(reader, start_block_number, start_offset); + + return result; +} + +/* Create a buffered writer for an index region starting at offset. */ +int uds_make_buffered_writer(struct io_factory *factory, off_t offset, u64 block_count, + struct buffered_writer **writer_ptr) +{ + int result; + struct dm_bufio_client *client = NULL; + struct buffered_writer *writer; + + result = uds_make_bufio(factory, offset, UDS_BLOCK_SIZE, 1, &client); + if (result != UDS_SUCCESS) + return result; + + result = vdo_allocate(1, struct buffered_writer, "buffered writer", &writer); + if (result != VDO_SUCCESS) { + dm_bufio_client_destroy(client); + return result; + } + + *writer = (struct buffered_writer) { + .factory = factory, + .client = client, + .buffer = NULL, + .limit = block_count, + .start = NULL, + .end = NULL, + .block_number = 0, + .error = UDS_SUCCESS, + }; + + uds_get_io_factory(factory); + *writer_ptr = writer; + return UDS_SUCCESS; +} + +static size_t get_remaining_write_space(struct buffered_writer *writer) +{ + return writer->start + UDS_BLOCK_SIZE - writer->end; +} + +static int __must_check prepare_next_buffer(struct buffered_writer *writer) +{ + struct dm_buffer *buffer = NULL; + void *data; + + if (writer->block_number >= writer->limit) { + writer->error = UDS_OUT_OF_RANGE; + return UDS_OUT_OF_RANGE; + } + + data = dm_bufio_new(writer->client, writer->block_number, &buffer); + if (IS_ERR(data)) { + writer->error = -PTR_ERR(data); + return writer->error; + } + + writer->buffer = buffer; + writer->start = data; + writer->end = data; + return UDS_SUCCESS; +} + +static int flush_previous_buffer(struct buffered_writer *writer) +{ + size_t available; + + if (writer->buffer == NULL) + return writer->error; + + if (writer->error == UDS_SUCCESS) { + available = get_remaining_write_space(writer); + + if (available > 0) + memset(writer->end, 0, available); + + dm_bufio_mark_buffer_dirty(writer->buffer); + } + + dm_bufio_release(writer->buffer); + writer->buffer = NULL; + writer->start = NULL; + writer->end = NULL; + writer->block_number++; + return writer->error; +} + +void uds_free_buffered_writer(struct buffered_writer *writer) +{ + int result; + + if (writer == NULL) + return; + + flush_previous_buffer(writer); + result = -dm_bufio_write_dirty_buffers(writer->client); + if (result != UDS_SUCCESS) + vdo_log_warning_strerror(result, "%s: failed to sync storage", __func__); + + dm_bufio_client_destroy(writer->client); + uds_put_io_factory(writer->factory); + vdo_free(writer); +} + +/* + * Append data to the buffer, writing as needed. If no data is provided, zeros are written instead. + * If a write error occurs, it is recorded and returned on every subsequent write attempt. + */ +int uds_write_to_buffered_writer(struct buffered_writer *writer, const u8 *data, + size_t length) +{ + int result = writer->error; + size_t chunk_size; + + while ((length > 0) && (result == UDS_SUCCESS)) { + if (writer->buffer == NULL) { + result = prepare_next_buffer(writer); + continue; + } + + chunk_size = min(length, get_remaining_write_space(writer)); + if (data == NULL) { + memset(writer->end, 0, chunk_size); + } else { + memcpy(writer->end, data, chunk_size); + data += chunk_size; + } + + length -= chunk_size; + writer->end += chunk_size; + + if (get_remaining_write_space(writer) == 0) + result = uds_flush_buffered_writer(writer); + } + + return result; +} + +int uds_flush_buffered_writer(struct buffered_writer *writer) +{ + if (writer->error != UDS_SUCCESS) + return writer->error; + + return flush_previous_buffer(writer); +} diff --git a/drivers/md/dm-vdo/indexer/io-factory.h b/drivers/md/dm-vdo/indexer/io-factory.h new file mode 100644 index 000000000000..7fb5a0616a79 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/io-factory.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_IO_FACTORY_H +#define UDS_IO_FACTORY_H + +#include + +/* + * The I/O factory manages all low-level I/O operations to the underlying storage device. Its main + * clients are the index layout and the volume. The buffered reader and buffered writer interfaces + * are helpers for accessing data in a contiguous range of storage blocks. + */ + +struct buffered_reader; +struct buffered_writer; + +struct io_factory; + +enum { + UDS_BLOCK_SIZE = 4096, + SECTORS_PER_BLOCK = UDS_BLOCK_SIZE >> SECTOR_SHIFT, +}; + +int __must_check uds_make_io_factory(struct block_device *bdev, + struct io_factory **factory_ptr); + +int __must_check uds_replace_storage(struct io_factory *factory, + struct block_device *bdev); + +void uds_put_io_factory(struct io_factory *factory); + +size_t __must_check uds_get_writable_size(struct io_factory *factory); + +int __must_check uds_make_bufio(struct io_factory *factory, off_t block_offset, + size_t block_size, unsigned int reserved_buffers, + struct dm_bufio_client **client_ptr); + +int __must_check uds_make_buffered_reader(struct io_factory *factory, off_t offset, + u64 block_count, + struct buffered_reader **reader_ptr); + +void uds_free_buffered_reader(struct buffered_reader *reader); + +int __must_check uds_read_from_buffered_reader(struct buffered_reader *reader, u8 *data, + size_t length); + +int __must_check uds_verify_buffered_data(struct buffered_reader *reader, const u8 *value, + size_t length); + +int __must_check uds_make_buffered_writer(struct io_factory *factory, off_t offset, + u64 block_count, + struct buffered_writer **writer_ptr); + +void uds_free_buffered_writer(struct buffered_writer *buffer); + +int __must_check uds_write_to_buffered_writer(struct buffered_writer *writer, + const u8 *data, size_t length); + +int __must_check uds_flush_buffered_writer(struct buffered_writer *writer); + +#endif /* UDS_IO_FACTORY_H */ diff --git a/drivers/md/dm-vdo/indexer/open-chapter.c b/drivers/md/dm-vdo/indexer/open-chapter.c new file mode 100644 index 000000000000..4a67bcadaae0 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/open-chapter.c @@ -0,0 +1,426 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "open-chapter.h" + +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "permassert.h" + +#include "config.h" +#include "hash-utils.h" + +/* + * Each index zone has a dedicated open chapter zone structure which gets an equal share of the + * open chapter space. Records are assigned to zones based on their record name. Within each zone, + * records are stored in an array in the order they arrive. Additionally, a reference to each + * record is stored in a hash table to help determine if a new record duplicates an existing one. + * If new metadata for an existing name arrives, the record is altered in place. The array of + * records is 1-based so that record number 0 can be used to indicate an unused hash slot. + * + * Deleted records are marked with a flag rather than actually removed to simplify hash table + * management. The array of deleted flags overlays the array of hash slots, but the flags are + * indexed by record number instead of by record name. The number of hash slots will always be a + * power of two that is greater than the number of records to be indexed, guaranteeing that hash + * insertion cannot fail, and that there are sufficient flags for all records. + * + * Once any open chapter zone fills its available space, the chapter is closed. The records from + * each zone are interleaved to attempt to preserve temporal locality and assigned to record pages. + * Empty or deleted records are replaced by copies of a valid record so that the record pages only + * contain valid records. The chapter then constructs a delta index which maps each record name to + * the record page on which that record can be found, which is split into index pages. These + * structures are then passed to the volume to be recorded on storage. + * + * When the index is saved, the open chapter records are saved in a single array, once again + * interleaved to attempt to preserve temporal locality. When the index is reloaded, there may be a + * different number of zones than previously, so the records must be parcelled out to their new + * zones. In addition, depending on the distribution of record names, a new zone may have more + * records than it has space. In this case, the latest records for that zone will be discarded. + */ + +static const u8 OPEN_CHAPTER_MAGIC[] = "ALBOC"; +static const u8 OPEN_CHAPTER_VERSION[] = "02.00"; + +#define OPEN_CHAPTER_MAGIC_LENGTH (sizeof(OPEN_CHAPTER_MAGIC) - 1) +#define OPEN_CHAPTER_VERSION_LENGTH (sizeof(OPEN_CHAPTER_VERSION) - 1) +#define LOAD_RATIO 2 + +static inline size_t records_size(const struct open_chapter_zone *open_chapter) +{ + return sizeof(struct uds_volume_record) * (1 + open_chapter->capacity); +} + +static inline size_t slots_size(size_t slot_count) +{ + return sizeof(struct open_chapter_zone_slot) * slot_count; +} + +int uds_make_open_chapter(const struct index_geometry *geometry, unsigned int zone_count, + struct open_chapter_zone **open_chapter_ptr) +{ + int result; + struct open_chapter_zone *open_chapter; + size_t capacity = geometry->records_per_chapter / zone_count; + size_t slot_count = (1 << bits_per(capacity * LOAD_RATIO)); + + result = vdo_allocate_extended(struct open_chapter_zone, slot_count, + struct open_chapter_zone_slot, "open chapter", + &open_chapter); + if (result != VDO_SUCCESS) + return result; + + open_chapter->slot_count = slot_count; + open_chapter->capacity = capacity; + result = vdo_allocate_cache_aligned(records_size(open_chapter), "record pages", + &open_chapter->records); + if (result != VDO_SUCCESS) { + uds_free_open_chapter(open_chapter); + return result; + } + + *open_chapter_ptr = open_chapter; + return UDS_SUCCESS; +} + +void uds_reset_open_chapter(struct open_chapter_zone *open_chapter) +{ + open_chapter->size = 0; + open_chapter->deletions = 0; + + memset(open_chapter->records, 0, records_size(open_chapter)); + memset(open_chapter->slots, 0, slots_size(open_chapter->slot_count)); +} + +static unsigned int probe_chapter_slots(struct open_chapter_zone *open_chapter, + const struct uds_record_name *name) +{ + struct uds_volume_record *record; + unsigned int slot_count = open_chapter->slot_count; + unsigned int slot = uds_name_to_hash_slot(name, slot_count); + unsigned int record_number; + unsigned int attempts = 1; + + while (true) { + record_number = open_chapter->slots[slot].record_number; + + /* + * If the hash slot is empty, we've reached the end of a chain without finding the + * record and should terminate the search. + */ + if (record_number == 0) + return slot; + + /* + * If the name of the record referenced by the slot matches and has not been + * deleted, then we've found the requested name. + */ + record = &open_chapter->records[record_number]; + if ((memcmp(&record->name, name, UDS_RECORD_NAME_SIZE) == 0) && + !open_chapter->slots[record_number].deleted) + return slot; + + /* + * Quadratic probing: advance the probe by 1, 2, 3, etc. and try again. This + * performs better than linear probing and works best for 2^N slots. + */ + slot = (slot + attempts++) % slot_count; + } +} + +void uds_search_open_chapter(struct open_chapter_zone *open_chapter, + const struct uds_record_name *name, + struct uds_record_data *metadata, bool *found) +{ + unsigned int slot; + unsigned int record_number; + + slot = probe_chapter_slots(open_chapter, name); + record_number = open_chapter->slots[slot].record_number; + if (record_number == 0) { + *found = false; + } else { + *found = true; + *metadata = open_chapter->records[record_number].data; + } +} + +/* Add a record to the open chapter zone and return the remaining space. */ +int uds_put_open_chapter(struct open_chapter_zone *open_chapter, + const struct uds_record_name *name, + const struct uds_record_data *metadata) +{ + unsigned int slot; + unsigned int record_number; + struct uds_volume_record *record; + + if (open_chapter->size >= open_chapter->capacity) + return 0; + + slot = probe_chapter_slots(open_chapter, name); + record_number = open_chapter->slots[slot].record_number; + + if (record_number == 0) { + record_number = ++open_chapter->size; + open_chapter->slots[slot].record_number = record_number; + } + + record = &open_chapter->records[record_number]; + record->name = *name; + record->data = *metadata; + + return open_chapter->capacity - open_chapter->size; +} + +void uds_remove_from_open_chapter(struct open_chapter_zone *open_chapter, + const struct uds_record_name *name) +{ + unsigned int slot; + unsigned int record_number; + + slot = probe_chapter_slots(open_chapter, name); + record_number = open_chapter->slots[slot].record_number; + + if (record_number > 0) { + open_chapter->slots[record_number].deleted = true; + open_chapter->deletions += 1; + } +} + +void uds_free_open_chapter(struct open_chapter_zone *open_chapter) +{ + if (open_chapter != NULL) { + vdo_free(open_chapter->records); + vdo_free(open_chapter); + } +} + +/* Map each record name to its record page number in the delta chapter index. */ +static int fill_delta_chapter_index(struct open_chapter_zone **chapter_zones, + unsigned int zone_count, + struct open_chapter_index *index, + struct uds_volume_record *collated_records) +{ + int result; + unsigned int records_per_chapter; + unsigned int records_per_page; + unsigned int record_index; + unsigned int records = 0; + u32 page_number; + unsigned int z; + int overflow_count = 0; + struct uds_volume_record *fill_record = NULL; + + /* + * The record pages should not have any empty space, so find a record with which to fill + * the chapter zone if it was closed early, and also to replace any deleted records. The + * last record in any filled zone is guaranteed to not have been deleted, so use one of + * those. + */ + for (z = 0; z < zone_count; z++) { + struct open_chapter_zone *zone = chapter_zones[z]; + + if (zone->size == zone->capacity) { + fill_record = &zone->records[zone->size]; + break; + } + } + + records_per_chapter = index->geometry->records_per_chapter; + records_per_page = index->geometry->records_per_page; + + for (records = 0; records < records_per_chapter; records++) { + struct uds_volume_record *record = &collated_records[records]; + struct open_chapter_zone *open_chapter; + + /* The record arrays in the zones are 1-based. */ + record_index = 1 + (records / zone_count); + page_number = records / records_per_page; + open_chapter = chapter_zones[records % zone_count]; + + /* Use the fill record in place of an unused record. */ + if (record_index > open_chapter->size || + open_chapter->slots[record_index].deleted) { + *record = *fill_record; + continue; + } + + *record = open_chapter->records[record_index]; + result = uds_put_open_chapter_index_record(index, &record->name, + page_number); + switch (result) { + case UDS_SUCCESS: + break; + case UDS_OVERFLOW: + overflow_count++; + break; + default: + vdo_log_error_strerror(result, + "failed to build open chapter index"); + return result; + } + } + + if (overflow_count > 0) + vdo_log_warning("Failed to add %d entries to chapter index", + overflow_count); + + return UDS_SUCCESS; +} + +int uds_close_open_chapter(struct open_chapter_zone **chapter_zones, + unsigned int zone_count, struct volume *volume, + struct open_chapter_index *chapter_index, + struct uds_volume_record *collated_records, + u64 virtual_chapter_number) +{ + int result; + + uds_empty_open_chapter_index(chapter_index, virtual_chapter_number); + result = fill_delta_chapter_index(chapter_zones, zone_count, chapter_index, + collated_records); + if (result != UDS_SUCCESS) + return result; + + return uds_write_chapter(volume, chapter_index, collated_records); +} + +int uds_save_open_chapter(struct uds_index *index, struct buffered_writer *writer) +{ + int result; + struct open_chapter_zone *open_chapter; + struct uds_volume_record *record; + u8 record_count_data[sizeof(u32)]; + u32 record_count = 0; + unsigned int record_index; + unsigned int z; + + result = uds_write_to_buffered_writer(writer, OPEN_CHAPTER_MAGIC, + OPEN_CHAPTER_MAGIC_LENGTH); + if (result != UDS_SUCCESS) + return result; + + result = uds_write_to_buffered_writer(writer, OPEN_CHAPTER_VERSION, + OPEN_CHAPTER_VERSION_LENGTH); + if (result != UDS_SUCCESS) + return result; + + for (z = 0; z < index->zone_count; z++) { + open_chapter = index->zones[z]->open_chapter; + record_count += open_chapter->size - open_chapter->deletions; + } + + put_unaligned_le32(record_count, record_count_data); + result = uds_write_to_buffered_writer(writer, record_count_data, + sizeof(record_count_data)); + if (result != UDS_SUCCESS) + return result; + + record_index = 1; + while (record_count > 0) { + for (z = 0; z < index->zone_count; z++) { + open_chapter = index->zones[z]->open_chapter; + if (record_index > open_chapter->size) + continue; + + if (open_chapter->slots[record_index].deleted) + continue; + + record = &open_chapter->records[record_index]; + result = uds_write_to_buffered_writer(writer, (u8 *) record, + sizeof(*record)); + if (result != UDS_SUCCESS) + return result; + + record_count--; + } + + record_index++; + } + + return uds_flush_buffered_writer(writer); +} + +u64 uds_compute_saved_open_chapter_size(struct index_geometry *geometry) +{ + unsigned int records_per_chapter = geometry->records_per_chapter; + + return OPEN_CHAPTER_MAGIC_LENGTH + OPEN_CHAPTER_VERSION_LENGTH + sizeof(u32) + + records_per_chapter * sizeof(struct uds_volume_record); +} + +static int load_version20(struct uds_index *index, struct buffered_reader *reader) +{ + int result; + u32 record_count; + u8 record_count_data[sizeof(u32)]; + struct uds_volume_record record; + + /* + * Track which zones cannot accept any more records. If the open chapter had a different + * number of zones previously, some new zones may have more records than they have space + * for. These overflow records will be discarded. + */ + bool full_flags[MAX_ZONES] = { + false, + }; + + result = uds_read_from_buffered_reader(reader, (u8 *) &record_count_data, + sizeof(record_count_data)); + if (result != UDS_SUCCESS) + return result; + + record_count = get_unaligned_le32(record_count_data); + while (record_count-- > 0) { + unsigned int zone = 0; + + result = uds_read_from_buffered_reader(reader, (u8 *) &record, + sizeof(record)); + if (result != UDS_SUCCESS) + return result; + + if (index->zone_count > 1) + zone = uds_get_volume_index_zone(index->volume_index, + &record.name); + + if (!full_flags[zone]) { + struct open_chapter_zone *open_chapter; + unsigned int remaining; + + open_chapter = index->zones[zone]->open_chapter; + remaining = uds_put_open_chapter(open_chapter, &record.name, + &record.data); + /* Do not allow any zone to fill completely. */ + full_flags[zone] = (remaining <= 1); + } + } + + return UDS_SUCCESS; +} + +int uds_load_open_chapter(struct uds_index *index, struct buffered_reader *reader) +{ + u8 version[OPEN_CHAPTER_VERSION_LENGTH]; + int result; + + result = uds_verify_buffered_data(reader, OPEN_CHAPTER_MAGIC, + OPEN_CHAPTER_MAGIC_LENGTH); + if (result != UDS_SUCCESS) + return result; + + result = uds_read_from_buffered_reader(reader, version, sizeof(version)); + if (result != UDS_SUCCESS) + return result; + + if (memcmp(OPEN_CHAPTER_VERSION, version, sizeof(version)) != 0) { + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "Invalid open chapter version: %.*s", + (int) sizeof(version), version); + } + + return load_version20(index, reader); +} diff --git a/drivers/md/dm-vdo/indexer/open-chapter.h b/drivers/md/dm-vdo/indexer/open-chapter.h new file mode 100644 index 000000000000..a4250bb19525 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/open-chapter.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_OPEN_CHAPTER_H +#define UDS_OPEN_CHAPTER_H + +#include "chapter-index.h" +#include "geometry.h" +#include "index.h" +#include "volume.h" + +/* + * The open chapter tracks the newest records in memory. Like the index as a whole, each open + * chapter is divided into a number of independent zones which are interleaved when the chapter is + * committed to the volume. + */ + +enum { + OPEN_CHAPTER_RECORD_NUMBER_BITS = 23, +}; + +struct open_chapter_zone_slot { + /* If non-zero, the record number addressed by this hash slot */ + unsigned int record_number : OPEN_CHAPTER_RECORD_NUMBER_BITS; + /* If true, the record at the index of this hash slot was deleted */ + bool deleted : 1; +} __packed; + +struct open_chapter_zone { + /* The maximum number of records that can be stored */ + unsigned int capacity; + /* The number of records stored */ + unsigned int size; + /* The number of deleted records */ + unsigned int deletions; + /* Array of chunk records, 1-based */ + struct uds_volume_record *records; + /* The number of slots in the hash table */ + unsigned int slot_count; + /* The hash table slots, referencing virtual record numbers */ + struct open_chapter_zone_slot slots[]; +}; + +int __must_check uds_make_open_chapter(const struct index_geometry *geometry, + unsigned int zone_count, + struct open_chapter_zone **open_chapter_ptr); + +void uds_reset_open_chapter(struct open_chapter_zone *open_chapter); + +void uds_search_open_chapter(struct open_chapter_zone *open_chapter, + const struct uds_record_name *name, + struct uds_record_data *metadata, bool *found); + +int __must_check uds_put_open_chapter(struct open_chapter_zone *open_chapter, + const struct uds_record_name *name, + const struct uds_record_data *metadata); + +void uds_remove_from_open_chapter(struct open_chapter_zone *open_chapter, + const struct uds_record_name *name); + +void uds_free_open_chapter(struct open_chapter_zone *open_chapter); + +int __must_check uds_close_open_chapter(struct open_chapter_zone **chapter_zones, + unsigned int zone_count, struct volume *volume, + struct open_chapter_index *chapter_index, + struct uds_volume_record *collated_records, + u64 virtual_chapter_number); + +int __must_check uds_save_open_chapter(struct uds_index *index, + struct buffered_writer *writer); + +int __must_check uds_load_open_chapter(struct uds_index *index, + struct buffered_reader *reader); + +u64 uds_compute_saved_open_chapter_size(struct index_geometry *geometry); + +#endif /* UDS_OPEN_CHAPTER_H */ diff --git a/drivers/md/dm-vdo/indexer/radix-sort.c b/drivers/md/dm-vdo/indexer/radix-sort.c new file mode 100644 index 000000000000..66b8c706a1ef --- /dev/null +++ b/drivers/md/dm-vdo/indexer/radix-sort.c @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "radix-sort.h" + +#include +#include + +#include "memory-alloc.h" +#include "string-utils.h" + +/* + * This implementation allocates one large object to do the sorting, which can be reused as many + * times as desired. The amount of memory required is logarithmically proportional to the number of + * keys to be sorted. + */ + +/* Piles smaller than this are handled with a simple insertion sort. */ +#define INSERTION_SORT_THRESHOLD 12 + +/* Sort keys are pointers to immutable fixed-length arrays of bytes. */ +typedef const u8 *sort_key_t; + +/* + * The keys are separated into piles based on the byte in each keys at the current offset, so the + * number of keys with each byte must be counted. + */ +struct histogram { + /* The number of non-empty bins */ + u16 used; + /* The index (key byte) of the first non-empty bin */ + u16 first; + /* The index (key byte) of the last non-empty bin */ + u16 last; + /* The number of occurrences of each specific byte */ + u32 size[256]; +}; + +/* + * Sub-tasks are manually managed on a stack, both for performance and to put a logarithmic bound + * on the stack space needed. + */ +struct task { + /* Pointer to the first key to sort. */ + sort_key_t *first_key; + /* Pointer to the last key to sort. */ + sort_key_t *last_key; + /* The offset into the key at which to continue sorting. */ + u16 offset; + /* The number of bytes remaining in the sort keys. */ + u16 length; +}; + +struct radix_sorter { + unsigned int count; + struct histogram bins; + sort_key_t *pile[256]; + struct task *end_of_stack; + struct task insertion_list[256]; + struct task stack[]; +}; + +/* Compare a segment of two fixed-length keys starting at an offset. */ +static inline int compare(sort_key_t key1, sort_key_t key2, u16 offset, u16 length) +{ + return memcmp(&key1[offset], &key2[offset], length); +} + +/* Insert the next unsorted key into an array of sorted keys. */ +static inline void insert_key(const struct task task, sort_key_t *next) +{ + /* Pull the unsorted key out, freeing up the array slot. */ + sort_key_t unsorted = *next; + + /* Compare the key to the preceding sorted entries, shifting down ones that are larger. */ + while ((--next >= task.first_key) && + (compare(unsorted, next[0], task.offset, task.length) < 0)) + next[1] = next[0]; + + /* Insert the key into the last slot that was cleared, sorting it. */ + next[1] = unsorted; +} + +/* + * Sort a range of key segments using an insertion sort. This simple sort is faster than the + * 256-way radix sort when the number of keys to sort is small. + */ +static inline void insertion_sort(const struct task task) +{ + sort_key_t *next; + + for (next = task.first_key + 1; next <= task.last_key; next++) + insert_key(task, next); +} + +/* Push a sorting task onto a task stack. */ +static inline void push_task(struct task **stack_pointer, sort_key_t *first_key, + u32 count, u16 offset, u16 length) +{ + struct task *task = (*stack_pointer)++; + + task->first_key = first_key; + task->last_key = &first_key[count - 1]; + task->offset = offset; + task->length = length; +} + +static inline void swap_keys(sort_key_t *a, sort_key_t *b) +{ + sort_key_t c = *a; + *a = *b; + *b = c; +} + +/* + * Count the number of times each byte value appears in the arrays of keys to sort at the current + * offset, keeping track of the number of non-empty bins, and the index of the first and last + * non-empty bin. + */ +static inline void measure_bins(const struct task task, struct histogram *bins) +{ + sort_key_t *key_ptr; + + /* + * Subtle invariant: bins->used and bins->size[] are zero because the sorting code clears + * it all out as it goes. Even though this structure is re-used, we don't need to pay to + * zero it before starting a new tally. + */ + bins->first = U8_MAX; + bins->last = 0; + + for (key_ptr = task.first_key; key_ptr <= task.last_key; key_ptr++) { + /* Increment the count for the byte in the key at the current offset. */ + u8 bin = (*key_ptr)[task.offset]; + u32 size = ++bins->size[bin]; + + /* Track non-empty bins. */ + if (size == 1) { + bins->used += 1; + if (bin < bins->first) + bins->first = bin; + + if (bin > bins->last) + bins->last = bin; + } + } +} + +/* + * Convert the bin sizes to pointers to where each pile goes. + * + * pile[0] = first_key + bin->size[0], + * pile[1] = pile[0] + bin->size[1], etc. + * + * After the keys are moved to the appropriate pile, we'll need to sort each of the piles by the + * next radix position. A new task is put on the stack for each pile containing lots of keys, or a + * new task is put on the list for each pile containing few keys. + * + * @stack: pointer the top of the stack + * @end_of_stack: the end of the stack + * @list: pointer the head of the list + * @pile: array for pointers to the end of each pile + * @bins: the histogram of the sizes of each pile + * @first_key: the first key of the stack + * @offset: the next radix position to sort by + * @length: the number of bytes remaining in the sort keys + * + * Return: UDS_SUCCESS or an error code + */ +static inline int push_bins(struct task **stack, struct task *end_of_stack, + struct task **list, sort_key_t *pile[], + struct histogram *bins, sort_key_t *first_key, + u16 offset, u16 length) +{ + sort_key_t *pile_start = first_key; + int bin; + + for (bin = bins->first; ; bin++) { + u32 size = bins->size[bin]; + + /* Skip empty piles. */ + if (size == 0) + continue; + + /* There's no need to sort empty keys. */ + if (length > 0) { + if (size > INSERTION_SORT_THRESHOLD) { + if (*stack >= end_of_stack) + return UDS_BAD_STATE; + + push_task(stack, pile_start, size, offset, length); + } else if (size > 1) { + push_task(list, pile_start, size, offset, length); + } + } + + pile_start += size; + pile[bin] = pile_start; + if (--bins->used == 0) + break; + } + + return UDS_SUCCESS; +} + +int uds_make_radix_sorter(unsigned int count, struct radix_sorter **sorter) +{ + int result; + unsigned int stack_size = count / INSERTION_SORT_THRESHOLD; + struct radix_sorter *radix_sorter; + + result = vdo_allocate_extended(struct radix_sorter, stack_size, struct task, + __func__, &radix_sorter); + if (result != VDO_SUCCESS) + return result; + + radix_sorter->count = count; + radix_sorter->end_of_stack = radix_sorter->stack + stack_size; + *sorter = radix_sorter; + return UDS_SUCCESS; +} + +void uds_free_radix_sorter(struct radix_sorter *sorter) +{ + vdo_free(sorter); +} + +/* + * Sort pointers to fixed-length keys (arrays of bytes) using a radix sort. The sort implementation + * is unstable, so the relative ordering of equal keys is not preserved. + */ +int uds_radix_sort(struct radix_sorter *sorter, const unsigned char *keys[], + unsigned int count, unsigned short length) +{ + struct task start; + struct histogram *bins = &sorter->bins; + sort_key_t **pile = sorter->pile; + struct task *task_stack = sorter->stack; + + /* All zero-length keys are identical and therefore already sorted. */ + if ((count == 0) || (length == 0)) + return UDS_SUCCESS; + + /* The initial task is to sort the entire length of all the keys. */ + start = (struct task) { + .first_key = keys, + .last_key = &keys[count - 1], + .offset = 0, + .length = length, + }; + + if (count <= INSERTION_SORT_THRESHOLD) { + insertion_sort(start); + return UDS_SUCCESS; + } + + if (count > sorter->count) + return UDS_INVALID_ARGUMENT; + + /* + * Repeatedly consume a sorting task from the stack and process it, pushing new sub-tasks + * onto the stack for each radix-sorted pile. When all tasks and sub-tasks have been + * processed, the stack will be empty and all the keys in the starting task will be fully + * sorted. + */ + for (*task_stack = start; task_stack >= sorter->stack; task_stack--) { + const struct task task = *task_stack; + struct task *insertion_task_list; + int result; + sort_key_t *fence; + sort_key_t *end; + + measure_bins(task, bins); + + /* + * Now that we know how large each bin is, generate pointers for each of the piles + * and push a new task to sort each pile by the next radix byte. + */ + insertion_task_list = sorter->insertion_list; + result = push_bins(&task_stack, sorter->end_of_stack, + &insertion_task_list, pile, bins, task.first_key, + task.offset + 1, task.length - 1); + if (result != UDS_SUCCESS) { + memset(bins, 0, sizeof(*bins)); + return result; + } + + /* Now bins->used is zero again. */ + + /* + * Don't bother processing the last pile: when piles 0..N-1 are all in place, then + * pile N must also be in place. + */ + end = task.last_key - bins->size[bins->last]; + bins->size[bins->last] = 0; + + for (fence = task.first_key; fence <= end; ) { + u8 bin; + sort_key_t key = *fence; + + /* + * The radix byte of the key tells us which pile it belongs in. Swap it for + * an unprocessed item just below that pile, and repeat. + */ + while (--pile[bin = key[task.offset]] > fence) + swap_keys(pile[bin], &key); + + /* + * The pile reached the fence. Put the key at the bottom of that pile, + * completing it, and advance the fence to the next pile. + */ + *fence = key; + fence += bins->size[bin]; + bins->size[bin] = 0; + } + + /* Now bins->size[] is all zero again. */ + + /* + * When the number of keys in a task gets small enough, it is faster to use an + * insertion sort than to keep subdividing into tiny piles. + */ + while (--insertion_task_list >= sorter->insertion_list) + insertion_sort(*insertion_task_list); + } + + return UDS_SUCCESS; +} diff --git a/drivers/md/dm-vdo/indexer/radix-sort.h b/drivers/md/dm-vdo/indexer/radix-sort.h new file mode 100644 index 000000000000..812949bc2cee --- /dev/null +++ b/drivers/md/dm-vdo/indexer/radix-sort.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_RADIX_SORT_H +#define UDS_RADIX_SORT_H + +/* + * Radix sort is implemented using an American Flag sort, an unstable, in-place 8-bit radix + * exchange sort. This is adapted from the algorithm in the paper by Peter M. McIlroy, Keith + * Bostic, and M. Douglas McIlroy, "Engineering Radix Sort". + * + * http://www.usenix.org/publications/compsystems/1993/win_mcilroy.pdf + */ + +struct radix_sorter; + +int __must_check uds_make_radix_sorter(unsigned int count, struct radix_sorter **sorter); + +void uds_free_radix_sorter(struct radix_sorter *sorter); + +int __must_check uds_radix_sort(struct radix_sorter *sorter, const unsigned char *keys[], + unsigned int count, unsigned short length); + +#endif /* UDS_RADIX_SORT_H */ diff --git a/drivers/md/dm-vdo/indexer/sparse-cache.c b/drivers/md/dm-vdo/indexer/sparse-cache.c new file mode 100644 index 000000000000..28920167827c --- /dev/null +++ b/drivers/md/dm-vdo/indexer/sparse-cache.c @@ -0,0 +1,624 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "sparse-cache.h" + +#include +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "chapter-index.h" +#include "config.h" +#include "index.h" + +/* + * Since the cache is small, it is implemented as a simple array of cache entries. Searching for a + * specific virtual chapter is implemented as a linear search. The cache replacement policy is + * least-recently-used (LRU). Again, the small size of the cache allows the LRU order to be + * maintained by shifting entries in an array list. + * + * Changing the contents of the cache requires the coordinated participation of all zone threads + * via the careful use of barrier messages sent to all the index zones by the triage queue worker + * thread. The critical invariant for coordination is that the cache membership must not change + * between updates, so that all calls to uds_sparse_cache_contains() from the zone threads must all + * receive the same results for every virtual chapter number. To ensure that critical invariant, + * state changes such as "that virtual chapter is no longer in the volume" and "skip searching that + * chapter because it has had too many cache misses" are represented separately from the cache + * membership information (the virtual chapter number). + * + * As a result of this invariant, we have the guarantee that every zone thread will call + * uds_update_sparse_cache() once and exactly once to request a chapter that is not in the cache, + * and the serialization of the barrier requests from the triage queue ensures they will all + * request the same chapter number. This means the only synchronization we need can be provided by + * a pair of thread barriers used only in the uds_update_sparse_cache() call, providing a critical + * section where a single zone thread can drive the cache update while all the other zone threads + * are known to be blocked, waiting in the second barrier. Outside that critical section, all the + * zone threads implicitly hold a shared lock. Inside it, the thread for zone zero holds an + * exclusive lock. No other threads may access or modify the cache entries. + * + * Chapter statistics must only be modified by a single thread, which is also the zone zero thread. + * All fields that might be frequently updated by that thread are kept in separate cache-aligned + * structures so they will not cause cache contention via "false sharing" with the fields that are + * frequently accessed by all of the zone threads. + * + * The LRU order is managed independently by each zone thread, and each zone uses its own list for + * searching and cache membership queries. The zone zero list is used to decide which chapter to + * evict when the cache is updated, and its search list is copied to the other threads at that + * time. + * + * The virtual chapter number field of the cache entry is the single field indicating whether a + * chapter is a member of the cache or not. The value NO_CHAPTER is used to represent a null or + * undefined chapter number. When present in the virtual chapter number field of a + * cached_chapter_index, it indicates that the cache entry is dead, and all the other fields of + * that entry (other than immutable pointers to cache memory) are undefined and irrelevant. Any + * cache entry that is not marked as dead is fully defined and a member of the cache, and + * uds_sparse_cache_contains() will always return true for any virtual chapter number that appears + * in any of the cache entries. + * + * A chapter index that is a member of the cache may be excluded from searches between calls to + * uds_update_sparse_cache() in two different ways. First, when a chapter falls off the end of the + * volume, its virtual chapter number will be less that the oldest virtual chapter number. Since + * that chapter is no longer part of the volume, there's no point in continuing to search that + * chapter index. Once invalidated, that virtual chapter will still be considered a member of the + * cache, but it will no longer be searched for matching names. + * + * The second mechanism is a heuristic based on keeping track of the number of consecutive search + * misses in a given chapter index. Once that count exceeds a threshold, the skip_search flag will + * be set to true, causing the chapter to be skipped when searching the entire cache, but still + * allowing it to be found when searching for a hook in that specific chapter. Finding a hook will + * clear the skip_search flag, once again allowing the non-hook searches to use that cache entry. + * Again, regardless of the state of the skip_search flag, the virtual chapter must still + * considered to be a member of the cache for uds_sparse_cache_contains(). + */ + +#define SKIP_SEARCH_THRESHOLD 20000 +#define ZONE_ZERO 0 + +/* + * These counters are essentially fields of the struct cached_chapter_index, but are segregated + * into this structure because they are frequently modified. They are grouped and aligned to keep + * them on different cache lines from the chapter fields that are accessed far more often than they + * are updated. + */ +struct __aligned(L1_CACHE_BYTES) cached_index_counters { + u64 consecutive_misses; +}; + +struct __aligned(L1_CACHE_BYTES) cached_chapter_index { + /* + * The virtual chapter number of the cached chapter index. NO_CHAPTER means this cache + * entry is unused. This field must only be modified in the critical section in + * uds_update_sparse_cache(). + */ + u64 virtual_chapter; + + u32 index_pages_count; + + /* + * These pointers are immutable during the life of the cache. The contents of the arrays + * change when the cache entry is replaced. + */ + struct delta_index_page *index_pages; + struct dm_buffer **page_buffers; + + /* + * If set, skip the chapter when searching the entire cache. This flag is just a + * performance optimization. This flag is mutable between cache updates, but it rarely + * changes and is frequently accessed, so it groups with the immutable fields. + */ + bool skip_search; + + /* + * The cache-aligned counters change often and are placed at the end of the structure to + * prevent false sharing with the more stable fields above. + */ + struct cached_index_counters counters; +}; + +/* + * A search_list represents an ordering of the sparse chapter index cache entry array, from most + * recently accessed to least recently accessed, which is the order in which the indexes should be + * searched and the reverse order in which they should be evicted from the cache. + * + * Cache entries that are dead or empty are kept at the end of the list, avoiding the need to even + * iterate over them to search, and ensuring that dead entries are replaced before any live entries + * are evicted. + * + * The search list is instantiated for each zone thread, avoiding any need for synchronization. The + * structure is allocated on a cache boundary to avoid false sharing of memory cache lines between + * zone threads. + */ +struct search_list { + u8 capacity; + u8 first_dead_entry; + struct cached_chapter_index *entries[]; +}; + +struct threads_barrier { + /* Lock for this barrier object */ + struct semaphore lock; + /* Semaphore for threads waiting at this barrier */ + struct semaphore wait; + /* Number of threads which have arrived */ + int arrived; + /* Total number of threads using this barrier */ + int thread_count; +}; + +struct sparse_cache { + const struct index_geometry *geometry; + unsigned int capacity; + unsigned int zone_count; + + unsigned int skip_threshold; + struct search_list *search_lists[MAX_ZONES]; + struct cached_chapter_index **scratch_entries; + + struct threads_barrier begin_update_barrier; + struct threads_barrier end_update_barrier; + + struct cached_chapter_index chapters[]; +}; + +static void initialize_threads_barrier(struct threads_barrier *barrier, + unsigned int thread_count) +{ + sema_init(&barrier->lock, 1); + barrier->arrived = 0; + barrier->thread_count = thread_count; + sema_init(&barrier->wait, 0); +} + +static inline void __down(struct semaphore *semaphore) +{ + /* + * Do not use down(semaphore). Instead use down_interruptible so that + * we do not get 120 second stall messages in kern.log. + */ + while (down_interruptible(semaphore) != 0) { + /* + * If we're called from a user-mode process (e.g., "dmsetup + * remove") while waiting for an operation that may take a + * while (e.g., UDS index save), and a signal is sent (SIGINT, + * SIGUSR2), then down_interruptible will not block. If that + * happens, sleep briefly to avoid keeping the CPU locked up in + * this loop. We could just call cond_resched, but then we'd + * still keep consuming CPU time slices and swamp other threads + * trying to do computational work. + */ + fsleep(1000); + } +} + +static void enter_threads_barrier(struct threads_barrier *barrier) +{ + __down(&barrier->lock); + if (++barrier->arrived == barrier->thread_count) { + /* last thread */ + int i; + + for (i = 1; i < barrier->thread_count; i++) + up(&barrier->wait); + + barrier->arrived = 0; + up(&barrier->lock); + } else { + up(&barrier->lock); + __down(&barrier->wait); + } +} + +static int __must_check initialize_cached_chapter_index(struct cached_chapter_index *chapter, + const struct index_geometry *geometry) +{ + int result; + + chapter->virtual_chapter = NO_CHAPTER; + chapter->index_pages_count = geometry->index_pages_per_chapter; + + result = vdo_allocate(chapter->index_pages_count, struct delta_index_page, + __func__, &chapter->index_pages); + if (result != VDO_SUCCESS) + return result; + + return vdo_allocate(chapter->index_pages_count, struct dm_buffer *, + "sparse index volume pages", &chapter->page_buffers); +} + +static int __must_check make_search_list(struct sparse_cache *cache, + struct search_list **list_ptr) +{ + struct search_list *list; + unsigned int bytes; + u8 i; + int result; + + bytes = (sizeof(struct search_list) + + (cache->capacity * sizeof(struct cached_chapter_index *))); + result = vdo_allocate_cache_aligned(bytes, "search list", &list); + if (result != VDO_SUCCESS) + return result; + + list->capacity = cache->capacity; + list->first_dead_entry = 0; + + for (i = 0; i < list->capacity; i++) + list->entries[i] = &cache->chapters[i]; + + *list_ptr = list; + return UDS_SUCCESS; +} + +int uds_make_sparse_cache(const struct index_geometry *geometry, unsigned int capacity, + unsigned int zone_count, struct sparse_cache **cache_ptr) +{ + int result; + unsigned int i; + struct sparse_cache *cache; + unsigned int bytes; + + bytes = (sizeof(struct sparse_cache) + (capacity * sizeof(struct cached_chapter_index))); + result = vdo_allocate_cache_aligned(bytes, "sparse cache", &cache); + if (result != VDO_SUCCESS) + return result; + + cache->geometry = geometry; + cache->capacity = capacity; + cache->zone_count = zone_count; + + /* + * Scale down the skip threshold since the cache only counts cache misses in zone zero, but + * requests are being handled in all zones. + */ + cache->skip_threshold = (SKIP_SEARCH_THRESHOLD / zone_count); + + initialize_threads_barrier(&cache->begin_update_barrier, zone_count); + initialize_threads_barrier(&cache->end_update_barrier, zone_count); + + for (i = 0; i < capacity; i++) { + result = initialize_cached_chapter_index(&cache->chapters[i], geometry); + if (result != UDS_SUCCESS) + goto out; + } + + for (i = 0; i < zone_count; i++) { + result = make_search_list(cache, &cache->search_lists[i]); + if (result != UDS_SUCCESS) + goto out; + } + + /* purge_search_list() needs some temporary lists for sorting. */ + result = vdo_allocate(capacity * 2, struct cached_chapter_index *, + "scratch entries", &cache->scratch_entries); + if (result != VDO_SUCCESS) + goto out; + + *cache_ptr = cache; + return UDS_SUCCESS; +out: + uds_free_sparse_cache(cache); + return result; +} + +static inline void set_skip_search(struct cached_chapter_index *chapter, + bool skip_search) +{ + /* Check before setting to reduce cache line contention. */ + if (READ_ONCE(chapter->skip_search) != skip_search) + WRITE_ONCE(chapter->skip_search, skip_search); +} + +static void score_search_hit(struct cached_chapter_index *chapter) +{ + chapter->counters.consecutive_misses = 0; + set_skip_search(chapter, false); +} + +static void score_search_miss(struct sparse_cache *cache, + struct cached_chapter_index *chapter) +{ + chapter->counters.consecutive_misses++; + if (chapter->counters.consecutive_misses > cache->skip_threshold) + set_skip_search(chapter, true); +} + +static void release_cached_chapter_index(struct cached_chapter_index *chapter) +{ + unsigned int i; + + chapter->virtual_chapter = NO_CHAPTER; + if (chapter->page_buffers == NULL) + return; + + for (i = 0; i < chapter->index_pages_count; i++) { + if (chapter->page_buffers[i] != NULL) + dm_bufio_release(vdo_forget(chapter->page_buffers[i])); + } +} + +void uds_free_sparse_cache(struct sparse_cache *cache) +{ + unsigned int i; + + if (cache == NULL) + return; + + vdo_free(cache->scratch_entries); + + for (i = 0; i < cache->zone_count; i++) + vdo_free(cache->search_lists[i]); + + for (i = 0; i < cache->capacity; i++) { + release_cached_chapter_index(&cache->chapters[i]); + vdo_free(cache->chapters[i].index_pages); + vdo_free(cache->chapters[i].page_buffers); + } + + vdo_free(cache); +} + +/* + * Take the indicated element of the search list and move it to the start, pushing the pointers + * previously before it back down the list. + */ +static inline void set_newest_entry(struct search_list *search_list, u8 index) +{ + struct cached_chapter_index *newest; + + if (index > 0) { + newest = search_list->entries[index]; + memmove(&search_list->entries[1], &search_list->entries[0], + index * sizeof(struct cached_chapter_index *)); + search_list->entries[0] = newest; + } + + /* + * This function may have moved a dead chapter to the front of the list for reuse, in which + * case the set of dead chapters becomes smaller. + */ + if (search_list->first_dead_entry <= index) + search_list->first_dead_entry++; +} + +bool uds_sparse_cache_contains(struct sparse_cache *cache, u64 virtual_chapter, + unsigned int zone_number) +{ + struct search_list *search_list; + struct cached_chapter_index *chapter; + u8 i; + + /* + * The correctness of the barriers depends on the invariant that between calls to + * uds_update_sparse_cache(), the answers this function returns must never vary: the result + * for a given chapter must be identical across zones. That invariant must be maintained + * even if the chapter falls off the end of the volume, or if searching it is disabled + * because of too many search misses. + */ + search_list = cache->search_lists[zone_number]; + for (i = 0; i < search_list->first_dead_entry; i++) { + chapter = search_list->entries[i]; + + if (virtual_chapter == chapter->virtual_chapter) { + if (zone_number == ZONE_ZERO) + score_search_hit(chapter); + + set_newest_entry(search_list, i); + return true; + } + } + + return false; +} + +/* + * Re-sort cache entries into three sets (active, skippable, and dead) while maintaining the LRU + * ordering that already existed. This operation must only be called during the critical section in + * uds_update_sparse_cache(). + */ +static void purge_search_list(struct search_list *search_list, + struct sparse_cache *cache, u64 oldest_virtual_chapter) +{ + struct cached_chapter_index **entries; + struct cached_chapter_index **skipped; + struct cached_chapter_index **dead; + struct cached_chapter_index *chapter; + unsigned int next_alive = 0; + unsigned int next_skipped = 0; + unsigned int next_dead = 0; + unsigned int i; + + entries = &search_list->entries[0]; + skipped = &cache->scratch_entries[0]; + dead = &cache->scratch_entries[search_list->capacity]; + + for (i = 0; i < search_list->first_dead_entry; i++) { + chapter = search_list->entries[i]; + if ((chapter->virtual_chapter < oldest_virtual_chapter) || + (chapter->virtual_chapter == NO_CHAPTER)) + dead[next_dead++] = chapter; + else if (chapter->skip_search) + skipped[next_skipped++] = chapter; + else + entries[next_alive++] = chapter; + } + + memcpy(&entries[next_alive], skipped, + next_skipped * sizeof(struct cached_chapter_index *)); + memcpy(&entries[next_alive + next_skipped], dead, + next_dead * sizeof(struct cached_chapter_index *)); + search_list->first_dead_entry = next_alive + next_skipped; +} + +static int __must_check cache_chapter_index(struct cached_chapter_index *chapter, + u64 virtual_chapter, + const struct volume *volume) +{ + int result; + + release_cached_chapter_index(chapter); + + result = uds_read_chapter_index_from_volume(volume, virtual_chapter, + chapter->page_buffers, + chapter->index_pages); + if (result != UDS_SUCCESS) + return result; + + chapter->counters.consecutive_misses = 0; + chapter->virtual_chapter = virtual_chapter; + chapter->skip_search = false; + + return UDS_SUCCESS; +} + +static inline void copy_search_list(const struct search_list *source, + struct search_list *target) +{ + *target = *source; + memcpy(target->entries, source->entries, + source->capacity * sizeof(struct cached_chapter_index *)); +} + +/* + * Update the sparse cache to contain a chapter index. This function must be called by all the zone + * threads with the same chapter number to correctly enter the thread barriers used to synchronize + * the cache updates. + */ +int uds_update_sparse_cache(struct index_zone *zone, u64 virtual_chapter) +{ + int result = UDS_SUCCESS; + const struct uds_index *index = zone->index; + struct sparse_cache *cache = index->volume->sparse_cache; + + if (uds_sparse_cache_contains(cache, virtual_chapter, zone->id)) + return UDS_SUCCESS; + + /* + * Wait for every zone thread to reach its corresponding barrier request and invoke this + * function before starting to modify the cache. + */ + enter_threads_barrier(&cache->begin_update_barrier); + + /* + * This is the start of the critical section: the zone zero thread is captain, effectively + * holding an exclusive lock on the sparse cache. All the other zone threads must do + * nothing between the two barriers. They will wait at the end_update_barrier again for the + * captain to finish the update. + */ + + if (zone->id == ZONE_ZERO) { + unsigned int z; + struct search_list *list = cache->search_lists[ZONE_ZERO]; + + purge_search_list(list, cache, zone->oldest_virtual_chapter); + + if (virtual_chapter >= index->oldest_virtual_chapter) { + set_newest_entry(list, list->capacity - 1); + result = cache_chapter_index(list->entries[0], virtual_chapter, + index->volume); + } + + for (z = 1; z < cache->zone_count; z++) + copy_search_list(list, cache->search_lists[z]); + } + + /* + * This is the end of the critical section. All cache invariants must have been restored. + */ + enter_threads_barrier(&cache->end_update_barrier); + return result; +} + +void uds_invalidate_sparse_cache(struct sparse_cache *cache) +{ + unsigned int i; + + for (i = 0; i < cache->capacity; i++) + release_cached_chapter_index(&cache->chapters[i]); +} + +static inline bool should_skip_chapter(struct cached_chapter_index *chapter, + u64 oldest_chapter, u64 requested_chapter) +{ + if ((chapter->virtual_chapter == NO_CHAPTER) || + (chapter->virtual_chapter < oldest_chapter)) + return true; + + if (requested_chapter != NO_CHAPTER) + return requested_chapter != chapter->virtual_chapter; + else + return READ_ONCE(chapter->skip_search); +} + +static int __must_check search_cached_chapter_index(struct cached_chapter_index *chapter, + const struct index_geometry *geometry, + const struct index_page_map *index_page_map, + const struct uds_record_name *name, + u16 *record_page_ptr) +{ + u32 physical_chapter = + uds_map_to_physical_chapter(geometry, chapter->virtual_chapter); + u32 index_page_number = + uds_find_index_page_number(index_page_map, name, physical_chapter); + struct delta_index_page *index_page = + &chapter->index_pages[index_page_number]; + + return uds_search_chapter_index_page(index_page, geometry, name, + record_page_ptr); +} + +int uds_search_sparse_cache(struct index_zone *zone, const struct uds_record_name *name, + u64 *virtual_chapter_ptr, u16 *record_page_ptr) +{ + int result; + struct volume *volume = zone->index->volume; + struct sparse_cache *cache = volume->sparse_cache; + struct cached_chapter_index *chapter; + struct search_list *search_list; + u8 i; + /* Search the entire cache unless a specific chapter was requested. */ + bool search_one = (*virtual_chapter_ptr != NO_CHAPTER); + + *record_page_ptr = NO_CHAPTER_INDEX_ENTRY; + search_list = cache->search_lists[zone->id]; + for (i = 0; i < search_list->first_dead_entry; i++) { + chapter = search_list->entries[i]; + + if (should_skip_chapter(chapter, zone->oldest_virtual_chapter, + *virtual_chapter_ptr)) + continue; + + result = search_cached_chapter_index(chapter, cache->geometry, + volume->index_page_map, name, + record_page_ptr); + if (result != UDS_SUCCESS) + return result; + + if (*record_page_ptr != NO_CHAPTER_INDEX_ENTRY) { + /* + * In theory, this might be a false match while a true match exists in + * another chapter, but that's a very rare case and not worth the extra + * search complexity. + */ + set_newest_entry(search_list, i); + if (zone->id == ZONE_ZERO) + score_search_hit(chapter); + + *virtual_chapter_ptr = chapter->virtual_chapter; + return UDS_SUCCESS; + } + + if (zone->id == ZONE_ZERO) + score_search_miss(cache, chapter); + + if (search_one) + break; + } + + return UDS_SUCCESS; +} diff --git a/drivers/md/dm-vdo/indexer/sparse-cache.h b/drivers/md/dm-vdo/indexer/sparse-cache.h new file mode 100644 index 000000000000..45e2dcf165b5 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/sparse-cache.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_SPARSE_CACHE_H +#define UDS_SPARSE_CACHE_H + +#include "geometry.h" +#include "indexer.h" + +/* + * The sparse cache is a cache of entire chapter indexes from sparse chapters used for searching + * for names after all other search paths have failed. It contains only complete chapter indexes; + * record pages from sparse chapters and single index pages used for resolving hooks are kept in + * the regular page cache in the volume. + * + * The most important property of this cache is the absence of synchronization for read operations. + * Safe concurrent access to the cache by the zone threads is controlled by the triage queue and + * the barrier requests it issues to the zone queues. The set of cached chapters does not and must + * not change between the carefully coordinated calls to uds_update_sparse_cache() from the zone + * threads. Outside of updates, every zone will get the same result when calling + * uds_sparse_cache_contains() as every other zone. + */ + +struct index_zone; +struct sparse_cache; + +int __must_check uds_make_sparse_cache(const struct index_geometry *geometry, + unsigned int capacity, unsigned int zone_count, + struct sparse_cache **cache_ptr); + +void uds_free_sparse_cache(struct sparse_cache *cache); + +bool uds_sparse_cache_contains(struct sparse_cache *cache, u64 virtual_chapter, + unsigned int zone_number); + +int __must_check uds_update_sparse_cache(struct index_zone *zone, u64 virtual_chapter); + +void uds_invalidate_sparse_cache(struct sparse_cache *cache); + +int __must_check uds_search_sparse_cache(struct index_zone *zone, + const struct uds_record_name *name, + u64 *virtual_chapter_ptr, u16 *record_page_ptr); + +#endif /* UDS_SPARSE_CACHE_H */ diff --git a/drivers/md/dm-vdo/indexer/volume-index.c b/drivers/md/dm-vdo/indexer/volume-index.c new file mode 100644 index 000000000000..12f954a0c532 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/volume-index.c @@ -0,0 +1,1283 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ +#include "volume-index.h" + +#include +#include +#include +#include +#include + +#include "errors.h" +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "permassert.h" +#include "thread-utils.h" + +#include "config.h" +#include "geometry.h" +#include "hash-utils.h" +#include "indexer.h" + +/* + * The volume index is a combination of two separate subindexes, one containing sparse hook entries + * (retained for all chapters), and one containing the remaining entries (retained only for the + * dense chapters). If there are no sparse chapters, only the non-hook sub index is used, and it + * will contain all records for all chapters. + * + * The volume index is also divided into zones, with one thread operating on each zone. Each + * incoming request is dispatched to the appropriate thread, and then to the appropriate subindex. + * Each delta list is handled by a single zone. To ensure that the distribution of delta lists to + * zones doesn't underflow (leaving some zone with no delta lists), the minimum number of delta + * lists must be the square of the maximum zone count for both subindexes. + * + * Each subindex zone is a delta index where the payload is a chapter number. The volume index can + * compute the delta list number, address, and zone number from the record name in order to + * dispatch record handling to the correct structures. + * + * Most operations that use all the zones take place either before request processing is allowed, + * or after all requests have been flushed in order to shut down. The only multi-threaded operation + * supported during normal operation is the uds_lookup_volume_index_name() method, used to determine + * whether a new chapter should be loaded into the sparse index cache. This operation only uses the + * sparse hook subindex, and the zone mutexes are used to make this operation safe. + * + * There are three ways of expressing chapter numbers in the volume index: virtual, index, and + * rolling. The interface to the volume index uses virtual chapter numbers, which are 64 bits long. + * Internally the subindex stores only the minimal number of bits necessary by masking away the + * high-order bits. When the index needs to deal with ordering of index chapter numbers, as when + * flushing entries from older chapters, it rolls the index chapter number around so that the + * smallest one in use is mapped to 0. See convert_index_to_virtual() or flush_invalid_entries() + * for an example of this technique. + * + * For efficiency, when older chapter numbers become invalid, the index does not immediately remove + * the invalidated entries. Instead it lazily removes them from a given delta list the next time it + * walks that list during normal operation. Because of this, the index size must be increased + * somewhat to accommodate all the invalid entries that have not yet been removed. For the standard + * index sizes, this requires about 4 chapters of old entries per 1024 chapters of valid entries in + * the index. + */ + +struct sub_index_parameters { + /* The number of bits in address mask */ + u8 address_bits; + /* The number of bits in chapter number */ + u8 chapter_bits; + /* The mean delta */ + u32 mean_delta; + /* The number of delta lists */ + u64 list_count; + /* The number of chapters used */ + u32 chapter_count; + /* The number of bits per chapter */ + size_t chapter_size_in_bits; + /* The number of bytes of delta list memory */ + size_t memory_size; + /* The number of bytes the index should keep free at all times */ + size_t target_free_bytes; +}; + +struct split_config { + /* The hook subindex configuration */ + struct uds_configuration hook_config; + struct index_geometry hook_geometry; + + /* The non-hook subindex configuration */ + struct uds_configuration non_hook_config; + struct index_geometry non_hook_geometry; +}; + +struct chapter_range { + u32 chapter_start; + u32 chapter_count; +}; + +#define MAGIC_SIZE 8 + +static const char MAGIC_START_5[] = "MI5-0005"; + +struct sub_index_data { + char magic[MAGIC_SIZE]; /* MAGIC_START_5 */ + u64 volume_nonce; + u64 virtual_chapter_low; + u64 virtual_chapter_high; + u32 first_list; + u32 list_count; +}; + +static const char MAGIC_START_6[] = "MI6-0001"; + +struct volume_index_data { + char magic[MAGIC_SIZE]; /* MAGIC_START_6 */ + u32 sparse_sample_rate; +}; + +static inline u32 extract_address(const struct volume_sub_index *sub_index, + const struct uds_record_name *name) +{ + return uds_extract_volume_index_bytes(name) & sub_index->address_mask; +} + +static inline u32 extract_dlist_num(const struct volume_sub_index *sub_index, + const struct uds_record_name *name) +{ + u64 bits = uds_extract_volume_index_bytes(name); + + return (bits >> sub_index->address_bits) % sub_index->list_count; +} + +static inline const struct volume_sub_index_zone * +get_zone_for_record(const struct volume_index_record *record) +{ + return &record->sub_index->zones[record->zone_number]; +} + +static inline u64 convert_index_to_virtual(const struct volume_index_record *record, + u32 index_chapter) +{ + const struct volume_sub_index_zone *volume_index_zone = get_zone_for_record(record); + u32 rolling_chapter = ((index_chapter - volume_index_zone->virtual_chapter_low) & + record->sub_index->chapter_mask); + + return volume_index_zone->virtual_chapter_low + rolling_chapter; +} + +static inline u32 convert_virtual_to_index(const struct volume_sub_index *sub_index, + u64 virtual_chapter) +{ + return virtual_chapter & sub_index->chapter_mask; +} + +static inline bool is_virtual_chapter_indexed(const struct volume_index_record *record, + u64 virtual_chapter) +{ + const struct volume_sub_index_zone *volume_index_zone = get_zone_for_record(record); + + return ((virtual_chapter >= volume_index_zone->virtual_chapter_low) && + (virtual_chapter <= volume_index_zone->virtual_chapter_high)); +} + +static inline bool has_sparse(const struct volume_index *volume_index) +{ + return volume_index->sparse_sample_rate > 0; +} + +bool uds_is_volume_index_sample(const struct volume_index *volume_index, + const struct uds_record_name *name) +{ + if (!has_sparse(volume_index)) + return false; + + return (uds_extract_sampling_bytes(name) % volume_index->sparse_sample_rate) == 0; +} + +static inline const struct volume_sub_index * +get_volume_sub_index(const struct volume_index *volume_index, + const struct uds_record_name *name) +{ + return (uds_is_volume_index_sample(volume_index, name) ? + &volume_index->vi_hook : + &volume_index->vi_non_hook); +} + +static unsigned int get_volume_sub_index_zone(const struct volume_sub_index *sub_index, + const struct uds_record_name *name) +{ + return extract_dlist_num(sub_index, name) / sub_index->delta_index.lists_per_zone; +} + +unsigned int uds_get_volume_index_zone(const struct volume_index *volume_index, + const struct uds_record_name *name) +{ + return get_volume_sub_index_zone(get_volume_sub_index(volume_index, name), name); +} + +#define DELTA_LIST_SIZE 256 + +static int compute_volume_sub_index_parameters(const struct uds_configuration *config, + struct sub_index_parameters *params) +{ + u64 entries_in_volume_index, address_span; + u32 chapters_in_volume_index, invalid_chapters; + u32 rounded_chapters; + u64 delta_list_records; + u32 address_count; + u64 index_size_in_bits; + size_t expected_index_size; + u64 min_delta_lists = MAX_ZONES * MAX_ZONES; + struct index_geometry *geometry = config->geometry; + u64 records_per_chapter = geometry->records_per_chapter; + + params->chapter_count = geometry->chapters_per_volume; + /* + * Make sure that the number of delta list records in the volume index does not change when + * the volume is reduced by one chapter. This preserves the mapping from name to volume + * index delta list. + */ + rounded_chapters = params->chapter_count; + if (uds_is_reduced_index_geometry(geometry)) + rounded_chapters += 1; + delta_list_records = records_per_chapter * rounded_chapters; + address_count = config->volume_index_mean_delta * DELTA_LIST_SIZE; + params->list_count = max(delta_list_records / DELTA_LIST_SIZE, min_delta_lists); + params->address_bits = bits_per(address_count - 1); + params->chapter_bits = bits_per(rounded_chapters - 1); + if ((u32) params->list_count != params->list_count) { + return vdo_log_warning_strerror(UDS_INVALID_ARGUMENT, + "cannot initialize volume index with %llu delta lists", + (unsigned long long) params->list_count); + } + + if (params->address_bits > 31) { + return vdo_log_warning_strerror(UDS_INVALID_ARGUMENT, + "cannot initialize volume index with %u address bits", + params->address_bits); + } + + /* + * The probability that a given delta list is not touched during the writing of an entire + * chapter is: + * + * double p_not_touched = pow((double) (params->list_count - 1) / params->list_count, + * records_per_chapter); + * + * For the standard index sizes, about 78% of the delta lists are not touched, and + * therefore contain old index entries that have not been eliminated by the lazy LRU + * processing. Then the number of old index entries that accumulate over the entire index, + * in terms of full chapters worth of entries, is: + * + * double invalid_chapters = p_not_touched / (1.0 - p_not_touched); + * + * For the standard index sizes, the index needs about 3.5 chapters of space for the old + * entries in a 1024 chapter index, so round this up to use 4 chapters per 1024 chapters in + * the index. + */ + invalid_chapters = max(rounded_chapters / 256, 2U); + chapters_in_volume_index = rounded_chapters + invalid_chapters; + entries_in_volume_index = records_per_chapter * chapters_in_volume_index; + + address_span = params->list_count << params->address_bits; + params->mean_delta = address_span / entries_in_volume_index; + + /* + * Compute the expected size of a full index, then set the total memory to be 6% larger + * than that expected size. This number should be large enough that there are not many + * rebalances when the index is full. + */ + params->chapter_size_in_bits = uds_compute_delta_index_size(records_per_chapter, + params->mean_delta, + params->chapter_bits); + index_size_in_bits = params->chapter_size_in_bits * chapters_in_volume_index; + expected_index_size = index_size_in_bits / BITS_PER_BYTE; + params->memory_size = expected_index_size * 106 / 100; + + params->target_free_bytes = expected_index_size / 20; + return UDS_SUCCESS; +} + +static void uninitialize_volume_sub_index(struct volume_sub_index *sub_index) +{ + vdo_free(vdo_forget(sub_index->flush_chapters)); + vdo_free(vdo_forget(sub_index->zones)); + uds_uninitialize_delta_index(&sub_index->delta_index); +} + +void uds_free_volume_index(struct volume_index *volume_index) +{ + if (volume_index == NULL) + return; + + if (volume_index->zones != NULL) + vdo_free(vdo_forget(volume_index->zones)); + + uninitialize_volume_sub_index(&volume_index->vi_non_hook); + uninitialize_volume_sub_index(&volume_index->vi_hook); + vdo_free(volume_index); +} + + +static int compute_volume_sub_index_save_bytes(const struct uds_configuration *config, + size_t *bytes) +{ + struct sub_index_parameters params = { .address_bits = 0 }; + int result; + + result = compute_volume_sub_index_parameters(config, ¶ms); + if (result != UDS_SUCCESS) + return result; + + *bytes = (sizeof(struct sub_index_data) + params.list_count * sizeof(u64) + + uds_compute_delta_index_save_bytes(params.list_count, + params.memory_size)); + return UDS_SUCCESS; +} + +/* This function is only useful if the configuration includes sparse chapters. */ +static void split_configuration(const struct uds_configuration *config, + struct split_config *split) +{ + u64 sample_rate, sample_records; + u64 dense_chapters, sparse_chapters; + + /* Start with copies of the base configuration. */ + split->hook_config = *config; + split->hook_geometry = *config->geometry; + split->hook_config.geometry = &split->hook_geometry; + split->non_hook_config = *config; + split->non_hook_geometry = *config->geometry; + split->non_hook_config.geometry = &split->non_hook_geometry; + + sample_rate = config->sparse_sample_rate; + sparse_chapters = config->geometry->sparse_chapters_per_volume; + dense_chapters = config->geometry->chapters_per_volume - sparse_chapters; + sample_records = config->geometry->records_per_chapter / sample_rate; + + /* Adjust the number of records indexed for each chapter. */ + split->hook_geometry.records_per_chapter = sample_records; + split->non_hook_geometry.records_per_chapter -= sample_records; + + /* Adjust the number of chapters indexed. */ + split->hook_geometry.sparse_chapters_per_volume = 0; + split->non_hook_geometry.sparse_chapters_per_volume = 0; + split->non_hook_geometry.chapters_per_volume = dense_chapters; +} + +static int compute_volume_index_save_bytes(const struct uds_configuration *config, + size_t *bytes) +{ + size_t hook_bytes, non_hook_bytes; + struct split_config split; + int result; + + if (!uds_is_sparse_index_geometry(config->geometry)) + return compute_volume_sub_index_save_bytes(config, bytes); + + split_configuration(config, &split); + result = compute_volume_sub_index_save_bytes(&split.hook_config, &hook_bytes); + if (result != UDS_SUCCESS) + return result; + + result = compute_volume_sub_index_save_bytes(&split.non_hook_config, + &non_hook_bytes); + if (result != UDS_SUCCESS) + return result; + + *bytes = sizeof(struct volume_index_data) + hook_bytes + non_hook_bytes; + return UDS_SUCCESS; +} + +int uds_compute_volume_index_save_blocks(const struct uds_configuration *config, + size_t block_size, u64 *block_count) +{ + size_t bytes; + int result; + + result = compute_volume_index_save_bytes(config, &bytes); + if (result != UDS_SUCCESS) + return result; + + bytes += sizeof(struct delta_list_save_info); + *block_count = DIV_ROUND_UP(bytes, block_size) + MAX_ZONES; + return UDS_SUCCESS; +} + +/* Flush invalid entries while walking the delta list. */ +static inline int flush_invalid_entries(struct volume_index_record *record, + struct chapter_range *flush_range, + u32 *next_chapter_to_invalidate) +{ + int result; + + result = uds_next_delta_index_entry(&record->delta_entry); + if (result != UDS_SUCCESS) + return result; + + while (!record->delta_entry.at_end) { + u32 index_chapter = uds_get_delta_entry_value(&record->delta_entry); + u32 relative_chapter = ((index_chapter - flush_range->chapter_start) & + record->sub_index->chapter_mask); + + if (likely(relative_chapter >= flush_range->chapter_count)) { + if (relative_chapter < *next_chapter_to_invalidate) + *next_chapter_to_invalidate = relative_chapter; + break; + } + + result = uds_remove_delta_index_entry(&record->delta_entry); + if (result != UDS_SUCCESS) + return result; + } + + return UDS_SUCCESS; +} + +/* Find the matching record, or the list offset where the record would go. */ +static int get_volume_index_entry(struct volume_index_record *record, u32 list_number, + u32 key, struct chapter_range *flush_range) +{ + struct volume_index_record other_record; + const struct volume_sub_index *sub_index = record->sub_index; + u32 next_chapter_to_invalidate = sub_index->chapter_mask; + int result; + + result = uds_start_delta_index_search(&sub_index->delta_index, list_number, 0, + &record->delta_entry); + if (result != UDS_SUCCESS) + return result; + + do { + result = flush_invalid_entries(record, flush_range, + &next_chapter_to_invalidate); + if (result != UDS_SUCCESS) + return result; + } while (!record->delta_entry.at_end && (key > record->delta_entry.key)); + + result = uds_remember_delta_index_offset(&record->delta_entry); + if (result != UDS_SUCCESS) + return result; + + /* Check any collision records for a more precise match. */ + other_record = *record; + if (!other_record.delta_entry.at_end && (key == other_record.delta_entry.key)) { + for (;;) { + u8 collision_name[UDS_RECORD_NAME_SIZE]; + + result = flush_invalid_entries(&other_record, flush_range, + &next_chapter_to_invalidate); + if (result != UDS_SUCCESS) + return result; + + if (other_record.delta_entry.at_end || + !other_record.delta_entry.is_collision) + break; + + result = uds_get_delta_entry_collision(&other_record.delta_entry, + collision_name); + if (result != UDS_SUCCESS) + return result; + + if (memcmp(collision_name, record->name, UDS_RECORD_NAME_SIZE) == 0) { + *record = other_record; + break; + } + } + } + while (!other_record.delta_entry.at_end) { + result = flush_invalid_entries(&other_record, flush_range, + &next_chapter_to_invalidate); + if (result != UDS_SUCCESS) + return result; + } + next_chapter_to_invalidate += flush_range->chapter_start; + next_chapter_to_invalidate &= sub_index->chapter_mask; + flush_range->chapter_start = next_chapter_to_invalidate; + flush_range->chapter_count = 0; + return UDS_SUCCESS; +} + +static int get_volume_sub_index_record(struct volume_sub_index *sub_index, + const struct uds_record_name *name, + struct volume_index_record *record) +{ + int result; + const struct volume_sub_index_zone *volume_index_zone; + u32 address = extract_address(sub_index, name); + u32 delta_list_number = extract_dlist_num(sub_index, name); + u64 flush_chapter = sub_index->flush_chapters[delta_list_number]; + + record->sub_index = sub_index; + record->mutex = NULL; + record->name = name; + record->zone_number = delta_list_number / sub_index->delta_index.lists_per_zone; + volume_index_zone = get_zone_for_record(record); + + if (flush_chapter < volume_index_zone->virtual_chapter_low) { + struct chapter_range range; + u64 flush_count = volume_index_zone->virtual_chapter_low - flush_chapter; + + range.chapter_start = convert_virtual_to_index(sub_index, flush_chapter); + range.chapter_count = (flush_count > sub_index->chapter_mask ? + sub_index->chapter_mask + 1 : + flush_count); + result = get_volume_index_entry(record, delta_list_number, address, + &range); + flush_chapter = convert_index_to_virtual(record, range.chapter_start); + if (flush_chapter > volume_index_zone->virtual_chapter_high) + flush_chapter = volume_index_zone->virtual_chapter_high; + sub_index->flush_chapters[delta_list_number] = flush_chapter; + } else { + result = uds_get_delta_index_entry(&sub_index->delta_index, + delta_list_number, address, + name->name, &record->delta_entry); + } + + if (result != UDS_SUCCESS) + return result; + + record->is_found = + (!record->delta_entry.at_end && (record->delta_entry.key == address)); + if (record->is_found) { + u32 index_chapter = uds_get_delta_entry_value(&record->delta_entry); + + record->virtual_chapter = convert_index_to_virtual(record, index_chapter); + } + + record->is_collision = record->delta_entry.is_collision; + return UDS_SUCCESS; +} + +int uds_get_volume_index_record(struct volume_index *volume_index, + const struct uds_record_name *name, + struct volume_index_record *record) +{ + int result; + + if (uds_is_volume_index_sample(volume_index, name)) { + /* + * Other threads cannot be allowed to call uds_lookup_volume_index_name() while + * this thread is finding the volume index record. Due to the lazy LRU flushing of + * the volume index, uds_get_volume_index_record() is not a read-only operation. + */ + unsigned int zone = + get_volume_sub_index_zone(&volume_index->vi_hook, name); + struct mutex *mutex = &volume_index->zones[zone].hook_mutex; + + mutex_lock(mutex); + result = get_volume_sub_index_record(&volume_index->vi_hook, name, + record); + mutex_unlock(mutex); + /* Remember the mutex so that other operations on the index record can use it. */ + record->mutex = mutex; + } else { + result = get_volume_sub_index_record(&volume_index->vi_non_hook, name, + record); + } + + return result; +} + +int uds_put_volume_index_record(struct volume_index_record *record, u64 virtual_chapter) +{ + int result; + u32 address; + const struct volume_sub_index *sub_index = record->sub_index; + + if (!is_virtual_chapter_indexed(record, virtual_chapter)) { + u64 low = get_zone_for_record(record)->virtual_chapter_low; + u64 high = get_zone_for_record(record)->virtual_chapter_high; + + return vdo_log_warning_strerror(UDS_INVALID_ARGUMENT, + "cannot put record into chapter number %llu that is out of the valid range %llu to %llu", + (unsigned long long) virtual_chapter, + (unsigned long long) low, + (unsigned long long) high); + } + address = extract_address(sub_index, record->name); + if (unlikely(record->mutex != NULL)) + mutex_lock(record->mutex); + result = uds_put_delta_index_entry(&record->delta_entry, address, + convert_virtual_to_index(sub_index, + virtual_chapter), + record->is_found ? record->name->name : NULL); + if (unlikely(record->mutex != NULL)) + mutex_unlock(record->mutex); + switch (result) { + case UDS_SUCCESS: + record->virtual_chapter = virtual_chapter; + record->is_collision = record->delta_entry.is_collision; + record->is_found = true; + break; + case UDS_OVERFLOW: + vdo_log_ratelimit(vdo_log_warning_strerror, UDS_OVERFLOW, + "Volume index entry dropped due to overflow condition"); + uds_log_delta_index_entry(&record->delta_entry); + break; + default: + break; + } + + return result; +} + +int uds_remove_volume_index_record(struct volume_index_record *record) +{ + int result; + + if (!record->is_found) + return vdo_log_warning_strerror(UDS_BAD_STATE, + "illegal operation on new record"); + + /* Mark the record so that it cannot be used again */ + record->is_found = false; + if (unlikely(record->mutex != NULL)) + mutex_lock(record->mutex); + result = uds_remove_delta_index_entry(&record->delta_entry); + if (unlikely(record->mutex != NULL)) + mutex_unlock(record->mutex); + return result; +} + +static void set_volume_sub_index_zone_open_chapter(struct volume_sub_index *sub_index, + unsigned int zone_number, + u64 virtual_chapter) +{ + u64 used_bits = 0; + struct volume_sub_index_zone *zone = &sub_index->zones[zone_number]; + struct delta_zone *delta_zone; + u32 i; + + zone->virtual_chapter_low = (virtual_chapter >= sub_index->chapter_count ? + virtual_chapter - sub_index->chapter_count + 1 : + 0); + zone->virtual_chapter_high = virtual_chapter; + + /* Check to see if the new zone data is too large. */ + delta_zone = &sub_index->delta_index.delta_zones[zone_number]; + for (i = 1; i <= delta_zone->list_count; i++) + used_bits += delta_zone->delta_lists[i].size; + + if (used_bits > sub_index->max_zone_bits) { + /* Expire enough chapters to free the desired space. */ + u64 expire_count = + 1 + (used_bits - sub_index->max_zone_bits) / sub_index->chapter_zone_bits; + + if (expire_count == 1) { + vdo_log_ratelimit(vdo_log_info, + "zone %u: At chapter %llu, expiring chapter %llu early", + zone_number, + (unsigned long long) virtual_chapter, + (unsigned long long) zone->virtual_chapter_low); + zone->early_flushes++; + zone->virtual_chapter_low++; + } else { + u64 first_expired = zone->virtual_chapter_low; + + if (first_expired + expire_count < zone->virtual_chapter_high) { + zone->early_flushes += expire_count; + zone->virtual_chapter_low += expire_count; + } else { + zone->early_flushes += + zone->virtual_chapter_high - zone->virtual_chapter_low; + zone->virtual_chapter_low = zone->virtual_chapter_high; + } + vdo_log_ratelimit(vdo_log_info, + "zone %u: At chapter %llu, expiring chapters %llu to %llu early", + zone_number, + (unsigned long long) virtual_chapter, + (unsigned long long) first_expired, + (unsigned long long) zone->virtual_chapter_low - 1); + } + } +} + +void uds_set_volume_index_zone_open_chapter(struct volume_index *volume_index, + unsigned int zone_number, + u64 virtual_chapter) +{ + struct mutex *mutex = &volume_index->zones[zone_number].hook_mutex; + + set_volume_sub_index_zone_open_chapter(&volume_index->vi_non_hook, zone_number, + virtual_chapter); + + /* + * Other threads cannot be allowed to call uds_lookup_volume_index_name() while the open + * chapter number is changing. + */ + if (has_sparse(volume_index)) { + mutex_lock(mutex); + set_volume_sub_index_zone_open_chapter(&volume_index->vi_hook, + zone_number, virtual_chapter); + mutex_unlock(mutex); + } +} + +/* + * Set the newest open chapter number for the index, while also advancing the oldest valid chapter + * number. + */ +void uds_set_volume_index_open_chapter(struct volume_index *volume_index, + u64 virtual_chapter) +{ + unsigned int zone; + + for (zone = 0; zone < volume_index->zone_count; zone++) + uds_set_volume_index_zone_open_chapter(volume_index, zone, virtual_chapter); +} + +int uds_set_volume_index_record_chapter(struct volume_index_record *record, + u64 virtual_chapter) +{ + const struct volume_sub_index *sub_index = record->sub_index; + int result; + + if (!record->is_found) + return vdo_log_warning_strerror(UDS_BAD_STATE, + "illegal operation on new record"); + + if (!is_virtual_chapter_indexed(record, virtual_chapter)) { + u64 low = get_zone_for_record(record)->virtual_chapter_low; + u64 high = get_zone_for_record(record)->virtual_chapter_high; + + return vdo_log_warning_strerror(UDS_INVALID_ARGUMENT, + "cannot set chapter number %llu that is out of the valid range %llu to %llu", + (unsigned long long) virtual_chapter, + (unsigned long long) low, + (unsigned long long) high); + } + + if (unlikely(record->mutex != NULL)) + mutex_lock(record->mutex); + result = uds_set_delta_entry_value(&record->delta_entry, + convert_virtual_to_index(sub_index, + virtual_chapter)); + if (unlikely(record->mutex != NULL)) + mutex_unlock(record->mutex); + if (result != UDS_SUCCESS) + return result; + + record->virtual_chapter = virtual_chapter; + return UDS_SUCCESS; +} + +static u64 lookup_volume_sub_index_name(const struct volume_sub_index *sub_index, + const struct uds_record_name *name) +{ + int result; + u32 address = extract_address(sub_index, name); + u32 delta_list_number = extract_dlist_num(sub_index, name); + unsigned int zone_number = get_volume_sub_index_zone(sub_index, name); + const struct volume_sub_index_zone *zone = &sub_index->zones[zone_number]; + u64 virtual_chapter; + u32 index_chapter; + u32 rolling_chapter; + struct delta_index_entry delta_entry; + + result = uds_get_delta_index_entry(&sub_index->delta_index, delta_list_number, + address, name->name, &delta_entry); + if (result != UDS_SUCCESS) + return NO_CHAPTER; + + if (delta_entry.at_end || (delta_entry.key != address)) + return NO_CHAPTER; + + index_chapter = uds_get_delta_entry_value(&delta_entry); + rolling_chapter = (index_chapter - zone->virtual_chapter_low) & sub_index->chapter_mask; + + virtual_chapter = zone->virtual_chapter_low + rolling_chapter; + if (virtual_chapter > zone->virtual_chapter_high) + return NO_CHAPTER; + + return virtual_chapter; +} + +/* Do a read-only lookup of the record name for sparse cache management. */ +u64 uds_lookup_volume_index_name(const struct volume_index *volume_index, + const struct uds_record_name *name) +{ + unsigned int zone_number = uds_get_volume_index_zone(volume_index, name); + struct mutex *mutex = &volume_index->zones[zone_number].hook_mutex; + u64 virtual_chapter; + + if (!uds_is_volume_index_sample(volume_index, name)) + return NO_CHAPTER; + + mutex_lock(mutex); + virtual_chapter = lookup_volume_sub_index_name(&volume_index->vi_hook, name); + mutex_unlock(mutex); + + return virtual_chapter; +} + +static void abort_restoring_volume_sub_index(struct volume_sub_index *sub_index) +{ + uds_reset_delta_index(&sub_index->delta_index); +} + +static void abort_restoring_volume_index(struct volume_index *volume_index) +{ + abort_restoring_volume_sub_index(&volume_index->vi_non_hook); + if (has_sparse(volume_index)) + abort_restoring_volume_sub_index(&volume_index->vi_hook); +} + +static int start_restoring_volume_sub_index(struct volume_sub_index *sub_index, + struct buffered_reader **readers, + unsigned int reader_count) +{ + unsigned int z; + int result; + u64 virtual_chapter_low = 0, virtual_chapter_high = 0; + unsigned int i; + + for (i = 0; i < reader_count; i++) { + struct sub_index_data header; + u8 buffer[sizeof(struct sub_index_data)]; + size_t offset = 0; + u32 j; + + result = uds_read_from_buffered_reader(readers[i], buffer, + sizeof(buffer)); + if (result != UDS_SUCCESS) { + return vdo_log_warning_strerror(result, + "failed to read volume index header"); + } + + memcpy(&header.magic, buffer, MAGIC_SIZE); + offset += MAGIC_SIZE; + decode_u64_le(buffer, &offset, &header.volume_nonce); + decode_u64_le(buffer, &offset, &header.virtual_chapter_low); + decode_u64_le(buffer, &offset, &header.virtual_chapter_high); + decode_u32_le(buffer, &offset, &header.first_list); + decode_u32_le(buffer, &offset, &header.list_count); + + result = VDO_ASSERT(offset == sizeof(buffer), + "%zu bytes decoded of %zu expected", offset, + sizeof(buffer)); + if (result != VDO_SUCCESS) + result = UDS_CORRUPT_DATA; + + if (memcmp(header.magic, MAGIC_START_5, MAGIC_SIZE) != 0) { + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, + "volume index file had bad magic number"); + } + + if (sub_index->volume_nonce == 0) { + sub_index->volume_nonce = header.volume_nonce; + } else if (header.volume_nonce != sub_index->volume_nonce) { + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, + "volume index volume nonce incorrect"); + } + + if (i == 0) { + virtual_chapter_low = header.virtual_chapter_low; + virtual_chapter_high = header.virtual_chapter_high; + } else if (virtual_chapter_high != header.virtual_chapter_high) { + u64 low = header.virtual_chapter_low; + u64 high = header.virtual_chapter_high; + + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, + "Inconsistent volume index zone files: Chapter range is [%llu,%llu], chapter range %d is [%llu,%llu]", + (unsigned long long) virtual_chapter_low, + (unsigned long long) virtual_chapter_high, + i, (unsigned long long) low, + (unsigned long long) high); + } else if (virtual_chapter_low < header.virtual_chapter_low) { + virtual_chapter_low = header.virtual_chapter_low; + } + + for (j = 0; j < header.list_count; j++) { + u8 decoded[sizeof(u64)]; + + result = uds_read_from_buffered_reader(readers[i], decoded, + sizeof(u64)); + if (result != UDS_SUCCESS) { + return vdo_log_warning_strerror(result, + "failed to read volume index flush ranges"); + } + + sub_index->flush_chapters[header.first_list + j] = + get_unaligned_le64(decoded); + } + } + + for (z = 0; z < sub_index->zone_count; z++) { + memset(&sub_index->zones[z], 0, sizeof(struct volume_sub_index_zone)); + sub_index->zones[z].virtual_chapter_low = virtual_chapter_low; + sub_index->zones[z].virtual_chapter_high = virtual_chapter_high; + } + + result = uds_start_restoring_delta_index(&sub_index->delta_index, readers, + reader_count); + if (result != UDS_SUCCESS) + return vdo_log_warning_strerror(result, "restoring delta index failed"); + + return UDS_SUCCESS; +} + +static int start_restoring_volume_index(struct volume_index *volume_index, + struct buffered_reader **buffered_readers, + unsigned int reader_count) +{ + unsigned int i; + int result; + + if (!has_sparse(volume_index)) { + return start_restoring_volume_sub_index(&volume_index->vi_non_hook, + buffered_readers, reader_count); + } + + for (i = 0; i < reader_count; i++) { + struct volume_index_data header; + u8 buffer[sizeof(struct volume_index_data)]; + size_t offset = 0; + + result = uds_read_from_buffered_reader(buffered_readers[i], buffer, + sizeof(buffer)); + if (result != UDS_SUCCESS) { + return vdo_log_warning_strerror(result, + "failed to read volume index header"); + } + + memcpy(&header.magic, buffer, MAGIC_SIZE); + offset += MAGIC_SIZE; + decode_u32_le(buffer, &offset, &header.sparse_sample_rate); + + result = VDO_ASSERT(offset == sizeof(buffer), + "%zu bytes decoded of %zu expected", offset, + sizeof(buffer)); + if (result != VDO_SUCCESS) + result = UDS_CORRUPT_DATA; + + if (memcmp(header.magic, MAGIC_START_6, MAGIC_SIZE) != 0) + return vdo_log_warning_strerror(UDS_CORRUPT_DATA, + "volume index file had bad magic number"); + + if (i == 0) { + volume_index->sparse_sample_rate = header.sparse_sample_rate; + } else if (volume_index->sparse_sample_rate != header.sparse_sample_rate) { + vdo_log_warning_strerror(UDS_CORRUPT_DATA, + "Inconsistent sparse sample rate in delta index zone files: %u vs. %u", + volume_index->sparse_sample_rate, + header.sparse_sample_rate); + return UDS_CORRUPT_DATA; + } + } + + result = start_restoring_volume_sub_index(&volume_index->vi_non_hook, + buffered_readers, reader_count); + if (result != UDS_SUCCESS) + return result; + + return start_restoring_volume_sub_index(&volume_index->vi_hook, buffered_readers, + reader_count); +} + +static int finish_restoring_volume_sub_index(struct volume_sub_index *sub_index, + struct buffered_reader **buffered_readers, + unsigned int reader_count) +{ + return uds_finish_restoring_delta_index(&sub_index->delta_index, + buffered_readers, reader_count); +} + +static int finish_restoring_volume_index(struct volume_index *volume_index, + struct buffered_reader **buffered_readers, + unsigned int reader_count) +{ + int result; + + result = finish_restoring_volume_sub_index(&volume_index->vi_non_hook, + buffered_readers, reader_count); + if ((result == UDS_SUCCESS) && has_sparse(volume_index)) { + result = finish_restoring_volume_sub_index(&volume_index->vi_hook, + buffered_readers, + reader_count); + } + + return result; +} + +int uds_load_volume_index(struct volume_index *volume_index, + struct buffered_reader **readers, unsigned int reader_count) +{ + int result; + + /* Start by reading the header section of the stream. */ + result = start_restoring_volume_index(volume_index, readers, reader_count); + if (result != UDS_SUCCESS) + return result; + + result = finish_restoring_volume_index(volume_index, readers, reader_count); + if (result != UDS_SUCCESS) { + abort_restoring_volume_index(volume_index); + return result; + } + + /* Check the final guard lists to make sure there is no extra data. */ + result = uds_check_guard_delta_lists(readers, reader_count); + if (result != UDS_SUCCESS) + abort_restoring_volume_index(volume_index); + + return result; +} + +static int start_saving_volume_sub_index(const struct volume_sub_index *sub_index, + unsigned int zone_number, + struct buffered_writer *buffered_writer) +{ + int result; + struct volume_sub_index_zone *volume_index_zone = &sub_index->zones[zone_number]; + u32 first_list = sub_index->delta_index.delta_zones[zone_number].first_list; + u32 list_count = sub_index->delta_index.delta_zones[zone_number].list_count; + u8 buffer[sizeof(struct sub_index_data)]; + size_t offset = 0; + u32 i; + + memcpy(buffer, MAGIC_START_5, MAGIC_SIZE); + offset += MAGIC_SIZE; + encode_u64_le(buffer, &offset, sub_index->volume_nonce); + encode_u64_le(buffer, &offset, volume_index_zone->virtual_chapter_low); + encode_u64_le(buffer, &offset, volume_index_zone->virtual_chapter_high); + encode_u32_le(buffer, &offset, first_list); + encode_u32_le(buffer, &offset, list_count); + + result = VDO_ASSERT(offset == sizeof(struct sub_index_data), + "%zu bytes of config written, of %zu expected", offset, + sizeof(struct sub_index_data)); + if (result != VDO_SUCCESS) + return result; + + result = uds_write_to_buffered_writer(buffered_writer, buffer, offset); + if (result != UDS_SUCCESS) + return vdo_log_warning_strerror(result, + "failed to write volume index header"); + + for (i = 0; i < list_count; i++) { + u8 encoded[sizeof(u64)]; + + put_unaligned_le64(sub_index->flush_chapters[first_list + i], &encoded); + result = uds_write_to_buffered_writer(buffered_writer, encoded, + sizeof(u64)); + if (result != UDS_SUCCESS) { + return vdo_log_warning_strerror(result, + "failed to write volume index flush ranges"); + } + } + + return uds_start_saving_delta_index(&sub_index->delta_index, zone_number, + buffered_writer); +} + +static int start_saving_volume_index(const struct volume_index *volume_index, + unsigned int zone_number, + struct buffered_writer *writer) +{ + u8 buffer[sizeof(struct volume_index_data)]; + size_t offset = 0; + int result; + + if (!has_sparse(volume_index)) { + return start_saving_volume_sub_index(&volume_index->vi_non_hook, + zone_number, writer); + } + + memcpy(buffer, MAGIC_START_6, MAGIC_SIZE); + offset += MAGIC_SIZE; + encode_u32_le(buffer, &offset, volume_index->sparse_sample_rate); + result = VDO_ASSERT(offset == sizeof(struct volume_index_data), + "%zu bytes of header written, of %zu expected", offset, + sizeof(struct volume_index_data)); + if (result != VDO_SUCCESS) + return result; + + result = uds_write_to_buffered_writer(writer, buffer, offset); + if (result != UDS_SUCCESS) { + vdo_log_warning_strerror(result, "failed to write volume index header"); + return result; + } + + result = start_saving_volume_sub_index(&volume_index->vi_non_hook, zone_number, + writer); + if (result != UDS_SUCCESS) + return result; + + return start_saving_volume_sub_index(&volume_index->vi_hook, zone_number, + writer); +} + +static int finish_saving_volume_sub_index(const struct volume_sub_index *sub_index, + unsigned int zone_number) +{ + return uds_finish_saving_delta_index(&sub_index->delta_index, zone_number); +} + +static int finish_saving_volume_index(const struct volume_index *volume_index, + unsigned int zone_number) +{ + int result; + + result = finish_saving_volume_sub_index(&volume_index->vi_non_hook, zone_number); + if ((result == UDS_SUCCESS) && has_sparse(volume_index)) + result = finish_saving_volume_sub_index(&volume_index->vi_hook, zone_number); + return result; +} + +int uds_save_volume_index(struct volume_index *volume_index, + struct buffered_writer **writers, unsigned int writer_count) +{ + int result = UDS_SUCCESS; + unsigned int zone; + + for (zone = 0; zone < writer_count; zone++) { + result = start_saving_volume_index(volume_index, zone, writers[zone]); + if (result != UDS_SUCCESS) + break; + + result = finish_saving_volume_index(volume_index, zone); + if (result != UDS_SUCCESS) + break; + + result = uds_write_guard_delta_list(writers[zone]); + if (result != UDS_SUCCESS) + break; + + result = uds_flush_buffered_writer(writers[zone]); + if (result != UDS_SUCCESS) + break; + } + + return result; +} + +static void get_volume_sub_index_stats(const struct volume_sub_index *sub_index, + struct volume_index_stats *stats) +{ + struct delta_index_stats dis; + unsigned int z; + + uds_get_delta_index_stats(&sub_index->delta_index, &dis); + stats->rebalance_time = dis.rebalance_time; + stats->rebalance_count = dis.rebalance_count; + stats->record_count = dis.record_count; + stats->collision_count = dis.collision_count; + stats->discard_count = dis.discard_count; + stats->overflow_count = dis.overflow_count; + stats->delta_lists = dis.list_count; + stats->early_flushes = 0; + for (z = 0; z < sub_index->zone_count; z++) + stats->early_flushes += sub_index->zones[z].early_flushes; +} + +void uds_get_volume_index_stats(const struct volume_index *volume_index, + struct volume_index_stats *stats) +{ + struct volume_index_stats sparse_stats; + + get_volume_sub_index_stats(&volume_index->vi_non_hook, stats); + if (!has_sparse(volume_index)) + return; + + get_volume_sub_index_stats(&volume_index->vi_hook, &sparse_stats); + stats->rebalance_time += sparse_stats.rebalance_time; + stats->rebalance_count += sparse_stats.rebalance_count; + stats->record_count += sparse_stats.record_count; + stats->collision_count += sparse_stats.collision_count; + stats->discard_count += sparse_stats.discard_count; + stats->overflow_count += sparse_stats.overflow_count; + stats->delta_lists += sparse_stats.delta_lists; + stats->early_flushes += sparse_stats.early_flushes; +} + +static int initialize_volume_sub_index(const struct uds_configuration *config, + u64 volume_nonce, u8 tag, + struct volume_sub_index *sub_index) +{ + struct sub_index_parameters params = { .address_bits = 0 }; + unsigned int zone_count = config->zone_count; + u64 available_bytes = 0; + unsigned int z; + int result; + + result = compute_volume_sub_index_parameters(config, ¶ms); + if (result != UDS_SUCCESS) + return result; + + sub_index->address_bits = params.address_bits; + sub_index->address_mask = (1u << params.address_bits) - 1; + sub_index->chapter_bits = params.chapter_bits; + sub_index->chapter_mask = (1u << params.chapter_bits) - 1; + sub_index->chapter_count = params.chapter_count; + sub_index->list_count = params.list_count; + sub_index->zone_count = zone_count; + sub_index->chapter_zone_bits = params.chapter_size_in_bits / zone_count; + sub_index->volume_nonce = volume_nonce; + + result = uds_initialize_delta_index(&sub_index->delta_index, zone_count, + params.list_count, params.mean_delta, + params.chapter_bits, params.memory_size, + tag); + if (result != UDS_SUCCESS) + return result; + + for (z = 0; z < sub_index->delta_index.zone_count; z++) + available_bytes += sub_index->delta_index.delta_zones[z].size; + available_bytes -= params.target_free_bytes; + sub_index->max_zone_bits = (available_bytes * BITS_PER_BYTE) / zone_count; + sub_index->memory_size = (sub_index->delta_index.memory_size + + sizeof(struct volume_sub_index) + + (params.list_count * sizeof(u64)) + + (zone_count * sizeof(struct volume_sub_index_zone))); + + /* The following arrays are initialized to all zeros. */ + result = vdo_allocate(params.list_count, u64, "first chapter to flush", + &sub_index->flush_chapters); + if (result != VDO_SUCCESS) + return result; + + return vdo_allocate(zone_count, struct volume_sub_index_zone, + "volume index zones", &sub_index->zones); +} + +int uds_make_volume_index(const struct uds_configuration *config, u64 volume_nonce, + struct volume_index **volume_index_ptr) +{ + struct split_config split; + unsigned int zone; + struct volume_index *volume_index; + int result; + + result = vdo_allocate(1, struct volume_index, "volume index", &volume_index); + if (result != VDO_SUCCESS) + return result; + + volume_index->zone_count = config->zone_count; + + if (!uds_is_sparse_index_geometry(config->geometry)) { + result = initialize_volume_sub_index(config, volume_nonce, 'm', + &volume_index->vi_non_hook); + if (result != UDS_SUCCESS) { + uds_free_volume_index(volume_index); + return result; + } + + volume_index->memory_size = volume_index->vi_non_hook.memory_size; + *volume_index_ptr = volume_index; + return UDS_SUCCESS; + } + + volume_index->sparse_sample_rate = config->sparse_sample_rate; + + result = vdo_allocate(config->zone_count, struct volume_index_zone, + "volume index zones", &volume_index->zones); + if (result != VDO_SUCCESS) { + uds_free_volume_index(volume_index); + return result; + } + + for (zone = 0; zone < config->zone_count; zone++) + mutex_init(&volume_index->zones[zone].hook_mutex); + + split_configuration(config, &split); + result = initialize_volume_sub_index(&split.non_hook_config, volume_nonce, 'd', + &volume_index->vi_non_hook); + if (result != UDS_SUCCESS) { + uds_free_volume_index(volume_index); + return vdo_log_error_strerror(result, + "Error creating non hook volume index"); + } + + result = initialize_volume_sub_index(&split.hook_config, volume_nonce, 's', + &volume_index->vi_hook); + if (result != UDS_SUCCESS) { + uds_free_volume_index(volume_index); + return vdo_log_error_strerror(result, + "Error creating hook volume index"); + } + + volume_index->memory_size = + volume_index->vi_non_hook.memory_size + volume_index->vi_hook.memory_size; + *volume_index_ptr = volume_index; + return UDS_SUCCESS; +} diff --git a/drivers/md/dm-vdo/indexer/volume-index.h b/drivers/md/dm-vdo/indexer/volume-index.h new file mode 100644 index 000000000000..583998c547b7 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/volume-index.h @@ -0,0 +1,193 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_VOLUME_INDEX_H +#define UDS_VOLUME_INDEX_H + +#include + +#include "thread-utils.h" + +#include "config.h" +#include "delta-index.h" +#include "indexer.h" + +/* + * The volume index is the primary top-level index for UDS. It contains records which map a record + * name to the chapter where a record with that name is stored. This mapping can definitively say + * when no record exists. However, because we only use a subset of the name for this index, it + * cannot definitively say that a record for the entry does exist. It can only say that if a record + * exists, it will be in a particular chapter. The request can then be dispatched to that chapter + * for further processing. + * + * If the volume_index_record does not actually match the record name, the index can store a more + * specific collision record to disambiguate the new entry from the existing one. Index entries are + * managed with volume_index_record structures. + */ + +#define NO_CHAPTER U64_MAX + +struct volume_index_stats { + /* Nanoseconds spent rebalancing */ + ktime_t rebalance_time; + /* Number of memory rebalances */ + u32 rebalance_count; + /* The number of records in the index */ + u64 record_count; + /* The number of collision records */ + u64 collision_count; + /* The number of records removed */ + u64 discard_count; + /* The number of UDS_OVERFLOWs detected */ + u64 overflow_count; + /* The number of delta lists */ + u32 delta_lists; + /* Number of early flushes */ + u64 early_flushes; +}; + +struct volume_sub_index_zone { + u64 virtual_chapter_low; + u64 virtual_chapter_high; + u64 early_flushes; +} __aligned(L1_CACHE_BYTES); + +struct volume_sub_index { + /* The delta index */ + struct delta_index delta_index; + /* The first chapter to be flushed in each zone */ + u64 *flush_chapters; + /* The zones */ + struct volume_sub_index_zone *zones; + /* The volume nonce */ + u64 volume_nonce; + /* Expected size of a chapter (per zone) */ + u64 chapter_zone_bits; + /* Maximum size of the index (per zone) */ + u64 max_zone_bits; + /* The number of bits in address mask */ + u8 address_bits; + /* Mask to get address within delta list */ + u32 address_mask; + /* The number of bits in chapter number */ + u8 chapter_bits; + /* The largest storable chapter number */ + u32 chapter_mask; + /* The number of chapters used */ + u32 chapter_count; + /* The number of delta lists */ + u32 list_count; + /* The number of zones */ + unsigned int zone_count; + /* The amount of memory allocated */ + u64 memory_size; +}; + +struct volume_index_zone { + /* Protects the sampled index in this zone */ + struct mutex hook_mutex; +} __aligned(L1_CACHE_BYTES); + +struct volume_index { + u32 sparse_sample_rate; + unsigned int zone_count; + u64 memory_size; + struct volume_sub_index vi_non_hook; + struct volume_sub_index vi_hook; + struct volume_index_zone *zones; +}; + +/* + * The volume_index_record structure is used to facilitate processing of a record name. A client + * first calls uds_get_volume_index_record() to find the volume index record for a record name. The + * fields of the record can then be examined to determine the state of the record. + * + * If is_found is false, then the index did not find an entry for the record name. Calling + * uds_put_volume_index_record() will insert a new entry for that name at the proper place. + * + * If is_found is true, then we did find an entry for the record name, and the virtual_chapter and + * is_collision fields reflect the entry found. Subsequently, a call to + * uds_remove_volume_index_record() will remove the entry, a call to + * uds_set_volume_index_record_chapter() will update the existing entry, and a call to + * uds_put_volume_index_record() will insert a new collision record after the existing entry. + */ +struct volume_index_record { + /* Public fields */ + + /* Chapter where the record info is found */ + u64 virtual_chapter; + /* This record is a collision */ + bool is_collision; + /* This record is the requested record */ + bool is_found; + + /* Private fields */ + + /* Zone that contains this name */ + unsigned int zone_number; + /* The volume index */ + struct volume_sub_index *sub_index; + /* Mutex for accessing this delta index entry in the hook index */ + struct mutex *mutex; + /* The record name to which this record refers */ + const struct uds_record_name *name; + /* The delta index entry for this record */ + struct delta_index_entry delta_entry; +}; + +int __must_check uds_make_volume_index(const struct uds_configuration *config, + u64 volume_nonce, + struct volume_index **volume_index); + +void uds_free_volume_index(struct volume_index *volume_index); + +int __must_check uds_compute_volume_index_save_blocks(const struct uds_configuration *config, + size_t block_size, + u64 *block_count); + +unsigned int __must_check uds_get_volume_index_zone(const struct volume_index *volume_index, + const struct uds_record_name *name); + +bool __must_check uds_is_volume_index_sample(const struct volume_index *volume_index, + const struct uds_record_name *name); + +/* + * This function is only used to manage sparse cache membership. Most requests should use + * uds_get_volume_index_record() to look up index records instead. + */ +u64 __must_check uds_lookup_volume_index_name(const struct volume_index *volume_index, + const struct uds_record_name *name); + +int __must_check uds_get_volume_index_record(struct volume_index *volume_index, + const struct uds_record_name *name, + struct volume_index_record *record); + +int __must_check uds_put_volume_index_record(struct volume_index_record *record, + u64 virtual_chapter); + +int __must_check uds_remove_volume_index_record(struct volume_index_record *record); + +int __must_check uds_set_volume_index_record_chapter(struct volume_index_record *record, + u64 virtual_chapter); + +void uds_set_volume_index_open_chapter(struct volume_index *volume_index, + u64 virtual_chapter); + +void uds_set_volume_index_zone_open_chapter(struct volume_index *volume_index, + unsigned int zone_number, + u64 virtual_chapter); + +int __must_check uds_load_volume_index(struct volume_index *volume_index, + struct buffered_reader **readers, + unsigned int reader_count); + +int __must_check uds_save_volume_index(struct volume_index *volume_index, + struct buffered_writer **writers, + unsigned int writer_count); + +void uds_get_volume_index_stats(const struct volume_index *volume_index, + struct volume_index_stats *stats); + +#endif /* UDS_VOLUME_INDEX_H */ diff --git a/drivers/md/dm-vdo/indexer/volume.c b/drivers/md/dm-vdo/indexer/volume.c new file mode 100644 index 000000000000..655453bb276b --- /dev/null +++ b/drivers/md/dm-vdo/indexer/volume.c @@ -0,0 +1,1693 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "volume.h" + +#include +#include +#include + +#include "errors.h" +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" +#include "string-utils.h" +#include "thread-utils.h" + +#include "chapter-index.h" +#include "config.h" +#include "geometry.h" +#include "hash-utils.h" +#include "index.h" +#include "sparse-cache.h" + +/* + * The first block of the volume layout is reserved for the volume header, which is no longer used. + * The remainder of the volume is divided into chapters consisting of several pages of records, and + * several pages of static index to use to find those records. The index pages are recorded first, + * followed by the record pages. The chapters are written in order as they are filled, so the + * volume storage acts as a circular log of the most recent chapters, with each new chapter + * overwriting the oldest saved one. + * + * When a new chapter is filled and closed, the records from that chapter are sorted and + * interleaved in approximate temporal order, and assigned to record pages. Then a static delta + * index is generated to store which record page contains each record. The in-memory index page map + * is also updated to indicate which delta lists fall on each chapter index page. This means that + * when a record is read, the volume only has to load a single index page and a single record page, + * rather than search the entire chapter. These index and record pages are written to storage, and + * the index pages are transferred to the page cache under the theory that the most recently + * written chapter is likely to be accessed again soon. + * + * When reading a record, the volume index will indicate which chapter should contain it. The + * volume uses the index page map to determine which chapter index page needs to be loaded, and + * then reads the relevant record page number from the chapter index. Both index and record pages + * are stored in a page cache when read for the common case that subsequent records need the same + * pages. The page cache evicts the least recently accessed entries when caching new pages. In + * addition, the volume uses dm-bufio to manage access to the storage, which may allow for + * additional caching depending on available system resources. + * + * Record requests are handled from cached pages when possible. If a page needs to be read, it is + * placed on a queue along with the request that wants to read it. Any requests for the same page + * that arrive while the read is pending are added to the queue entry. A separate reader thread + * handles the queued reads, adding the page to the cache and updating any requests queued with it + * so they can continue processing. This allows the index zone threads to continue processing new + * requests rather than wait for the storage reads. + * + * When an index rebuild is necessary, the volume reads each stored chapter to determine which + * range of chapters contain valid records, so that those records can be used to reconstruct the + * in-memory volume index. + */ + +/* The maximum allowable number of contiguous bad chapters */ +#define MAX_BAD_CHAPTERS 100 +#define VOLUME_CACHE_MAX_ENTRIES (U16_MAX >> 1) +#define VOLUME_CACHE_QUEUED_FLAG (1 << 15) +#define VOLUME_CACHE_MAX_QUEUED_READS 4096 + +static const u64 BAD_CHAPTER = U64_MAX; + +/* + * The invalidate counter is two 32 bits fields stored together atomically. The low order 32 bits + * are the physical page number of the cached page being read. The high order 32 bits are a + * sequence number. This value is written when the zone that owns it begins or completes a cache + * search. Any other thread will only read the counter in wait_for_pending_searches() while waiting + * to update the cache contents. + */ +union invalidate_counter { + u64 value; + struct { + u32 page; + u32 counter; + }; +}; + +static inline u32 map_to_page_number(struct index_geometry *geometry, u32 physical_page) +{ + return (physical_page - HEADER_PAGES_PER_VOLUME) % geometry->pages_per_chapter; +} + +static inline u32 map_to_chapter_number(struct index_geometry *geometry, u32 physical_page) +{ + return (physical_page - HEADER_PAGES_PER_VOLUME) / geometry->pages_per_chapter; +} + +static inline bool is_record_page(struct index_geometry *geometry, u32 physical_page) +{ + return map_to_page_number(geometry, physical_page) >= geometry->index_pages_per_chapter; +} + +static u32 map_to_physical_page(const struct index_geometry *geometry, u32 chapter, u32 page) +{ + /* Page zero is the header page, so the first chapter index page is page one. */ + return HEADER_PAGES_PER_VOLUME + (geometry->pages_per_chapter * chapter) + page; +} + +static inline union invalidate_counter get_invalidate_counter(struct page_cache *cache, + unsigned int zone_number) +{ + return (union invalidate_counter) { + .value = READ_ONCE(cache->search_pending_counters[zone_number].atomic_value), + }; +} + +static inline void set_invalidate_counter(struct page_cache *cache, + unsigned int zone_number, + union invalidate_counter invalidate_counter) +{ + WRITE_ONCE(cache->search_pending_counters[zone_number].atomic_value, + invalidate_counter.value); +} + +static inline bool search_pending(union invalidate_counter invalidate_counter) +{ + return (invalidate_counter.counter & 1) != 0; +} + +/* Lock the cache for a zone in order to search for a page. */ +static void begin_pending_search(struct page_cache *cache, u32 physical_page, + unsigned int zone_number) +{ + union invalidate_counter invalidate_counter = + get_invalidate_counter(cache, zone_number); + + invalidate_counter.page = physical_page; + invalidate_counter.counter++; + set_invalidate_counter(cache, zone_number, invalidate_counter); + VDO_ASSERT_LOG_ONLY(search_pending(invalidate_counter), + "Search is pending for zone %u", zone_number); + /* + * This memory barrier ensures that the write to the invalidate counter is seen by other + * threads before this thread accesses the cached page. The corresponding read memory + * barrier is in wait_for_pending_searches(). + */ + smp_mb(); +} + +/* Unlock the cache for a zone by clearing its invalidate counter. */ +static void end_pending_search(struct page_cache *cache, unsigned int zone_number) +{ + union invalidate_counter invalidate_counter; + + /* + * This memory barrier ensures that this thread completes reads of the + * cached page before other threads see the write to the invalidate + * counter. + */ + smp_mb(); + + invalidate_counter = get_invalidate_counter(cache, zone_number); + VDO_ASSERT_LOG_ONLY(search_pending(invalidate_counter), + "Search is pending for zone %u", zone_number); + invalidate_counter.counter++; + set_invalidate_counter(cache, zone_number, invalidate_counter); +} + +static void wait_for_pending_searches(struct page_cache *cache, u32 physical_page) +{ + union invalidate_counter initial_counters[MAX_ZONES]; + unsigned int i; + + /* + * We hold the read_threads_mutex. We are waiting for threads that do not hold the + * read_threads_mutex. Those threads have "locked" their targeted page by setting the + * search_pending_counter. The corresponding write memory barrier is in + * begin_pending_search(). + */ + smp_mb(); + + for (i = 0; i < cache->zone_count; i++) + initial_counters[i] = get_invalidate_counter(cache, i); + for (i = 0; i < cache->zone_count; i++) { + if (search_pending(initial_counters[i]) && + (initial_counters[i].page == physical_page)) { + /* + * There is an active search using the physical page. We need to wait for + * the search to finish. + * + * FIXME: Investigate using wait_event() to wait for the search to finish. + */ + while (initial_counters[i].value == + get_invalidate_counter(cache, i).value) + cond_resched(); + } + } +} + +static void release_page_buffer(struct cached_page *page) +{ + if (page->buffer != NULL) + dm_bufio_release(vdo_forget(page->buffer)); +} + +static void clear_cache_page(struct page_cache *cache, struct cached_page *page) +{ + /* Do not clear read_pending because the read queue relies on it. */ + release_page_buffer(page); + page->physical_page = cache->indexable_pages; + WRITE_ONCE(page->last_used, 0); +} + +static void make_page_most_recent(struct page_cache *cache, struct cached_page *page) +{ + /* + * ASSERTION: We are either a zone thread holding a search_pending_counter, or we are any + * thread holding the read_threads_mutex. + */ + if (atomic64_read(&cache->clock) != READ_ONCE(page->last_used)) + WRITE_ONCE(page->last_used, atomic64_inc_return(&cache->clock)); +} + +/* Select a page to remove from the cache to make space for a new entry. */ +static struct cached_page *select_victim_in_cache(struct page_cache *cache) +{ + struct cached_page *page; + int oldest_index = 0; + s64 oldest_time = S64_MAX; + s64 last_used; + u16 i; + + /* Find the oldest unclaimed page. We hold the read_threads_mutex. */ + for (i = 0; i < cache->cache_slots; i++) { + /* A page with a pending read must not be replaced. */ + if (cache->cache[i].read_pending) + continue; + + last_used = READ_ONCE(cache->cache[i].last_used); + if (last_used <= oldest_time) { + oldest_time = last_used; + oldest_index = i; + } + } + + page = &cache->cache[oldest_index]; + if (page->physical_page != cache->indexable_pages) { + WRITE_ONCE(cache->index[page->physical_page], cache->cache_slots); + wait_for_pending_searches(cache, page->physical_page); + } + + page->read_pending = true; + clear_cache_page(cache, page); + return page; +} + +/* Make a newly filled cache entry available to other threads. */ +static int put_page_in_cache(struct page_cache *cache, u32 physical_page, + struct cached_page *page) +{ + int result; + + /* We hold the read_threads_mutex. */ + result = VDO_ASSERT((page->read_pending), "page to install has a pending read"); + if (result != VDO_SUCCESS) + return result; + + page->physical_page = physical_page; + make_page_most_recent(cache, page); + page->read_pending = false; + + /* + * We hold the read_threads_mutex, but we must have a write memory barrier before making + * the cached_page available to the readers that do not hold the mutex. The corresponding + * read memory barrier is in get_page_and_index(). + */ + smp_wmb(); + + /* This assignment also clears the queued flag. */ + WRITE_ONCE(cache->index[physical_page], page - cache->cache); + return UDS_SUCCESS; +} + +static void cancel_page_in_cache(struct page_cache *cache, u32 physical_page, + struct cached_page *page) +{ + int result; + + /* We hold the read_threads_mutex. */ + result = VDO_ASSERT((page->read_pending), "page to install has a pending read"); + if (result != VDO_SUCCESS) + return; + + clear_cache_page(cache, page); + page->read_pending = false; + + /* Clear the mapping and the queued flag for the new page. */ + WRITE_ONCE(cache->index[physical_page], cache->cache_slots); +} + +static inline u16 next_queue_position(u16 position) +{ + return (position + 1) % VOLUME_CACHE_MAX_QUEUED_READS; +} + +static inline void advance_queue_position(u16 *position) +{ + *position = next_queue_position(*position); +} + +static inline bool read_queue_is_full(struct page_cache *cache) +{ + return cache->read_queue_first == next_queue_position(cache->read_queue_last); +} + +static bool enqueue_read(struct page_cache *cache, struct uds_request *request, + u32 physical_page) +{ + struct queued_read *queue_entry; + u16 last = cache->read_queue_last; + u16 read_queue_index; + + /* We hold the read_threads_mutex. */ + if ((cache->index[physical_page] & VOLUME_CACHE_QUEUED_FLAG) == 0) { + /* This page has no existing entry in the queue. */ + if (read_queue_is_full(cache)) + return false; + + /* Fill in the read queue entry. */ + cache->read_queue[last].physical_page = physical_page; + cache->read_queue[last].invalid = false; + cache->read_queue[last].first_request = NULL; + cache->read_queue[last].last_request = NULL; + + /* Point the cache index to the read queue entry. */ + read_queue_index = last; + WRITE_ONCE(cache->index[physical_page], + read_queue_index | VOLUME_CACHE_QUEUED_FLAG); + + advance_queue_position(&cache->read_queue_last); + } else { + /* It's already queued, so add this request to the existing entry. */ + read_queue_index = cache->index[physical_page] & ~VOLUME_CACHE_QUEUED_FLAG; + } + + request->next_request = NULL; + queue_entry = &cache->read_queue[read_queue_index]; + if (queue_entry->first_request == NULL) + queue_entry->first_request = request; + else + queue_entry->last_request->next_request = request; + queue_entry->last_request = request; + + return true; +} + +static void enqueue_page_read(struct volume *volume, struct uds_request *request, + u32 physical_page) +{ + /* Mark the page as queued, so that chapter invalidation knows to cancel a read. */ + while (!enqueue_read(&volume->page_cache, request, physical_page)) { + vdo_log_debug("Read queue full, waiting for reads to finish"); + uds_wait_cond(&volume->read_threads_read_done_cond, + &volume->read_threads_mutex); + } + + uds_signal_cond(&volume->read_threads_cond); +} + +/* + * Reserve the next read queue entry for processing, but do not actually remove it from the queue. + * Must be followed by release_queued_requests(). + */ +static struct queued_read *reserve_read_queue_entry(struct page_cache *cache) +{ + /* We hold the read_threads_mutex. */ + struct queued_read *entry; + u16 index_value; + bool queued; + + /* No items to dequeue */ + if (cache->read_queue_next_read == cache->read_queue_last) + return NULL; + + entry = &cache->read_queue[cache->read_queue_next_read]; + index_value = cache->index[entry->physical_page]; + queued = (index_value & VOLUME_CACHE_QUEUED_FLAG) != 0; + /* Check to see if it's still queued before resetting. */ + if (entry->invalid && queued) + WRITE_ONCE(cache->index[entry->physical_page], cache->cache_slots); + + /* + * If a synchronous read has taken this page, set invalid to true so it doesn't get + * overwritten. Requests will just be requeued. + */ + if (!queued) + entry->invalid = true; + + entry->reserved = true; + advance_queue_position(&cache->read_queue_next_read); + return entry; +} + +static inline struct queued_read *wait_to_reserve_read_queue_entry(struct volume *volume) +{ + struct queued_read *queue_entry = NULL; + + while (!volume->read_threads_exiting) { + queue_entry = reserve_read_queue_entry(&volume->page_cache); + if (queue_entry != NULL) + break; + + uds_wait_cond(&volume->read_threads_cond, &volume->read_threads_mutex); + } + + return queue_entry; +} + +static int init_chapter_index_page(const struct volume *volume, u8 *index_page, + u32 chapter, u32 index_page_number, + struct delta_index_page *chapter_index_page) +{ + u64 ci_virtual; + u32 ci_chapter; + u32 lowest_list; + u32 highest_list; + struct index_geometry *geometry = volume->geometry; + int result; + + result = uds_initialize_chapter_index_page(chapter_index_page, geometry, + index_page, volume->nonce); + if (volume->lookup_mode == LOOKUP_FOR_REBUILD) + return result; + + if (result != UDS_SUCCESS) { + return vdo_log_error_strerror(result, + "Reading chapter index page for chapter %u page %u", + chapter, index_page_number); + } + + uds_get_list_number_bounds(volume->index_page_map, chapter, index_page_number, + &lowest_list, &highest_list); + ci_virtual = chapter_index_page->virtual_chapter_number; + ci_chapter = uds_map_to_physical_chapter(geometry, ci_virtual); + if ((chapter == ci_chapter) && + (lowest_list == chapter_index_page->lowest_list_number) && + (highest_list == chapter_index_page->highest_list_number)) + return UDS_SUCCESS; + + vdo_log_warning("Index page map updated to %llu", + (unsigned long long) volume->index_page_map->last_update); + vdo_log_warning("Page map expects that chapter %u page %u has range %u to %u, but chapter index page has chapter %llu with range %u to %u", + chapter, index_page_number, lowest_list, highest_list, + (unsigned long long) ci_virtual, + chapter_index_page->lowest_list_number, + chapter_index_page->highest_list_number); + return vdo_log_error_strerror(UDS_CORRUPT_DATA, + "index page map mismatch with chapter index"); +} + +static int initialize_index_page(const struct volume *volume, u32 physical_page, + struct cached_page *page) +{ + u32 chapter = map_to_chapter_number(volume->geometry, physical_page); + u32 index_page_number = map_to_page_number(volume->geometry, physical_page); + + return init_chapter_index_page(volume, dm_bufio_get_block_data(page->buffer), + chapter, index_page_number, &page->index_page); +} + +static bool search_record_page(const u8 record_page[], + const struct uds_record_name *name, + const struct index_geometry *geometry, + struct uds_record_data *metadata) +{ + /* + * The array of records is sorted by name and stored as a binary tree in heap order, so the + * root of the tree is the first array element. + */ + u32 node = 0; + const struct uds_volume_record *records = (const struct uds_volume_record *) record_page; + + while (node < geometry->records_per_page) { + int result; + const struct uds_volume_record *record = &records[node]; + + result = memcmp(name, &record->name, UDS_RECORD_NAME_SIZE); + if (result == 0) { + if (metadata != NULL) + *metadata = record->data; + return true; + } + + /* The children of node N are at indexes 2N+1 and 2N+2. */ + node = ((2 * node) + ((result < 0) ? 1 : 2)); + } + + return false; +} + +/* + * If we've read in a record page, we're going to do an immediate search, to speed up processing by + * avoiding get_record_from_zone(), and to ensure that requests make progress even when queued. If + * we've read in an index page, we save the record page number so we don't have to resolve the + * index page again. We use the location, virtual_chapter, and old_metadata fields in the request + * to allow the index code to know where to begin processing the request again. + */ +static int search_page(struct cached_page *page, const struct volume *volume, + struct uds_request *request, u32 physical_page) +{ + int result; + enum uds_index_region location; + u16 record_page_number; + + if (is_record_page(volume->geometry, physical_page)) { + if (search_record_page(dm_bufio_get_block_data(page->buffer), + &request->record_name, volume->geometry, + &request->old_metadata)) + location = UDS_LOCATION_RECORD_PAGE_LOOKUP; + else + location = UDS_LOCATION_UNAVAILABLE; + } else { + result = uds_search_chapter_index_page(&page->index_page, + volume->geometry, + &request->record_name, + &record_page_number); + if (result != UDS_SUCCESS) + return result; + + if (record_page_number == NO_CHAPTER_INDEX_ENTRY) { + location = UDS_LOCATION_UNAVAILABLE; + } else { + location = UDS_LOCATION_INDEX_PAGE_LOOKUP; + *((u16 *) &request->old_metadata) = record_page_number; + } + } + + request->location = location; + request->found = false; + return UDS_SUCCESS; +} + +static int process_entry(struct volume *volume, struct queued_read *entry) +{ + u32 page_number = entry->physical_page; + struct uds_request *request; + struct cached_page *page = NULL; + u8 *page_data; + int result; + + if (entry->invalid) { + vdo_log_debug("Requeuing requests for invalid page"); + return UDS_SUCCESS; + } + + page = select_victim_in_cache(&volume->page_cache); + + mutex_unlock(&volume->read_threads_mutex); + page_data = dm_bufio_read(volume->client, page_number, &page->buffer); + mutex_lock(&volume->read_threads_mutex); + if (IS_ERR(page_data)) { + result = -PTR_ERR(page_data); + vdo_log_warning_strerror(result, + "error reading physical page %u from volume", + page_number); + cancel_page_in_cache(&volume->page_cache, page_number, page); + return result; + } + + if (entry->invalid) { + vdo_log_warning("Page %u invalidated after read", page_number); + cancel_page_in_cache(&volume->page_cache, page_number, page); + return UDS_SUCCESS; + } + + if (!is_record_page(volume->geometry, page_number)) { + result = initialize_index_page(volume, page_number, page); + if (result != UDS_SUCCESS) { + vdo_log_warning("Error initializing chapter index page"); + cancel_page_in_cache(&volume->page_cache, page_number, page); + return result; + } + } + + result = put_page_in_cache(&volume->page_cache, page_number, page); + if (result != UDS_SUCCESS) { + vdo_log_warning("Error putting page %u in cache", page_number); + cancel_page_in_cache(&volume->page_cache, page_number, page); + return result; + } + + request = entry->first_request; + while ((request != NULL) && (result == UDS_SUCCESS)) { + result = search_page(page, volume, request, page_number); + request = request->next_request; + } + + return result; +} + +static void release_queued_requests(struct volume *volume, struct queued_read *entry, + int result) +{ + struct page_cache *cache = &volume->page_cache; + u16 next_read = cache->read_queue_next_read; + struct uds_request *request; + struct uds_request *next; + + for (request = entry->first_request; request != NULL; request = next) { + next = request->next_request; + request->status = result; + request->requeued = true; + uds_enqueue_request(request, STAGE_INDEX); + } + + entry->reserved = false; + + /* Move the read_queue_first pointer as far as we can. */ + while ((cache->read_queue_first != next_read) && + (!cache->read_queue[cache->read_queue_first].reserved)) + advance_queue_position(&cache->read_queue_first); + uds_broadcast_cond(&volume->read_threads_read_done_cond); +} + +static void read_thread_function(void *arg) +{ + struct volume *volume = arg; + + vdo_log_debug("reader starting"); + mutex_lock(&volume->read_threads_mutex); + while (true) { + struct queued_read *queue_entry; + int result; + + queue_entry = wait_to_reserve_read_queue_entry(volume); + if (volume->read_threads_exiting) + break; + + result = process_entry(volume, queue_entry); + release_queued_requests(volume, queue_entry, result); + } + mutex_unlock(&volume->read_threads_mutex); + vdo_log_debug("reader done"); +} + +static void get_page_and_index(struct page_cache *cache, u32 physical_page, + int *queue_index, struct cached_page **page_ptr) +{ + u16 index_value; + u16 index; + bool queued; + + /* + * ASSERTION: We are either a zone thread holding a search_pending_counter, or we are any + * thread holding the read_threads_mutex. + * + * Holding only a search_pending_counter is the most frequent case. + */ + /* + * It would be unlikely for the compiler to turn the usage of index_value into two reads of + * cache->index, but it would be possible and very bad if those reads did not return the + * same bits. + */ + index_value = READ_ONCE(cache->index[physical_page]); + queued = (index_value & VOLUME_CACHE_QUEUED_FLAG) != 0; + index = index_value & ~VOLUME_CACHE_QUEUED_FLAG; + + if (!queued && (index < cache->cache_slots)) { + *page_ptr = &cache->cache[index]; + /* + * We have acquired access to the cached page, but unless we hold the + * read_threads_mutex, we need a read memory barrier now. The corresponding write + * memory barrier is in put_page_in_cache(). + */ + smp_rmb(); + } else { + *page_ptr = NULL; + } + + *queue_index = queued ? index : -1; +} + +static void get_page_from_cache(struct page_cache *cache, u32 physical_page, + struct cached_page **page) +{ + /* + * ASSERTION: We are in a zone thread. + * ASSERTION: We holding a search_pending_counter or the read_threads_mutex. + */ + int queue_index = -1; + + get_page_and_index(cache, physical_page, &queue_index, page); +} + +static int read_page_locked(struct volume *volume, u32 physical_page, + struct cached_page **page_ptr) +{ + int result = UDS_SUCCESS; + struct cached_page *page = NULL; + u8 *page_data; + + page = select_victim_in_cache(&volume->page_cache); + page_data = dm_bufio_read(volume->client, physical_page, &page->buffer); + if (IS_ERR(page_data)) { + result = -PTR_ERR(page_data); + vdo_log_warning_strerror(result, + "error reading physical page %u from volume", + physical_page); + cancel_page_in_cache(&volume->page_cache, physical_page, page); + return result; + } + + if (!is_record_page(volume->geometry, physical_page)) { + result = initialize_index_page(volume, physical_page, page); + if (result != UDS_SUCCESS) { + if (volume->lookup_mode != LOOKUP_FOR_REBUILD) + vdo_log_warning("Corrupt index page %u", physical_page); + cancel_page_in_cache(&volume->page_cache, physical_page, page); + return result; + } + } + + result = put_page_in_cache(&volume->page_cache, physical_page, page); + if (result != UDS_SUCCESS) { + vdo_log_warning("Error putting page %u in cache", physical_page); + cancel_page_in_cache(&volume->page_cache, physical_page, page); + return result; + } + + *page_ptr = page; + return UDS_SUCCESS; +} + +/* Retrieve a page from the cache while holding the read threads mutex. */ +static int get_volume_page_locked(struct volume *volume, u32 physical_page, + struct cached_page **page_ptr) +{ + int result; + struct cached_page *page = NULL; + + get_page_from_cache(&volume->page_cache, physical_page, &page); + if (page == NULL) { + result = read_page_locked(volume, physical_page, &page); + if (result != UDS_SUCCESS) + return result; + } else { + make_page_most_recent(&volume->page_cache, page); + } + + *page_ptr = page; + return UDS_SUCCESS; +} + +/* Retrieve a page from the cache while holding a search_pending lock. */ +static int get_volume_page_protected(struct volume *volume, struct uds_request *request, + u32 physical_page, struct cached_page **page_ptr) +{ + struct cached_page *page; + + get_page_from_cache(&volume->page_cache, physical_page, &page); + if (page != NULL) { + if (request->zone_number == 0) { + /* Only one zone is allowed to update the LRU. */ + make_page_most_recent(&volume->page_cache, page); + } + + *page_ptr = page; + return UDS_SUCCESS; + } + + /* Prepare to enqueue a read for the page. */ + end_pending_search(&volume->page_cache, request->zone_number); + mutex_lock(&volume->read_threads_mutex); + + /* + * Do the lookup again while holding the read mutex (no longer the fast case so this should + * be fine to repeat). We need to do this because a page may have been added to the cache + * by a reader thread between the time we searched above and the time we went to actually + * try to enqueue it below. This could result in us enqueuing another read for a page which + * is already in the cache, which would mean we end up with two entries in the cache for + * the same page. + */ + get_page_from_cache(&volume->page_cache, physical_page, &page); + if (page == NULL) { + enqueue_page_read(volume, request, physical_page); + /* + * The performance gain from unlocking first, while "search pending" mode is off, + * turns out to be significant in some cases. The page is not available yet so + * the order does not matter for correctness as it does below. + */ + mutex_unlock(&volume->read_threads_mutex); + begin_pending_search(&volume->page_cache, physical_page, + request->zone_number); + return UDS_QUEUED; + } + + /* + * Now that the page is loaded, the volume needs to switch to "reader thread unlocked" and + * "search pending" state in careful order so no other thread can mess with the data before + * the caller gets to look at it. + */ + begin_pending_search(&volume->page_cache, physical_page, request->zone_number); + mutex_unlock(&volume->read_threads_mutex); + *page_ptr = page; + return UDS_SUCCESS; +} + +static int get_volume_page(struct volume *volume, u32 chapter, u32 page_number, + struct cached_page **page_ptr) +{ + int result; + u32 physical_page = map_to_physical_page(volume->geometry, chapter, page_number); + + mutex_lock(&volume->read_threads_mutex); + result = get_volume_page_locked(volume, physical_page, page_ptr); + mutex_unlock(&volume->read_threads_mutex); + return result; +} + +int uds_get_volume_record_page(struct volume *volume, u32 chapter, u32 page_number, + u8 **data_ptr) +{ + int result; + struct cached_page *page = NULL; + + result = get_volume_page(volume, chapter, page_number, &page); + if (result == UDS_SUCCESS) + *data_ptr = dm_bufio_get_block_data(page->buffer); + return result; +} + +int uds_get_volume_index_page(struct volume *volume, u32 chapter, u32 page_number, + struct delta_index_page **index_page_ptr) +{ + int result; + struct cached_page *page = NULL; + + result = get_volume_page(volume, chapter, page_number, &page); + if (result == UDS_SUCCESS) + *index_page_ptr = &page->index_page; + return result; +} + +/* + * Find the record page associated with a name in a given index page. This will return UDS_QUEUED + * if the page in question must be read from storage. + */ +static int search_cached_index_page(struct volume *volume, struct uds_request *request, + u32 chapter, u32 index_page_number, + u16 *record_page_number) +{ + int result; + struct cached_page *page = NULL; + u32 physical_page = map_to_physical_page(volume->geometry, chapter, + index_page_number); + + /* + * Make sure the invalidate counter is updated before we try and read the mapping. This + * prevents this thread from reading a page in the cache which has already been marked for + * invalidation by the reader thread, before the reader thread has noticed that the + * invalidate_counter has been incremented. + */ + begin_pending_search(&volume->page_cache, physical_page, request->zone_number); + + result = get_volume_page_protected(volume, request, physical_page, &page); + if (result != UDS_SUCCESS) { + end_pending_search(&volume->page_cache, request->zone_number); + return result; + } + + result = uds_search_chapter_index_page(&page->index_page, volume->geometry, + &request->record_name, + record_page_number); + end_pending_search(&volume->page_cache, request->zone_number); + return result; +} + +/* + * Find the metadata associated with a name in a given record page. This will return UDS_QUEUED if + * the page in question must be read from storage. + */ +int uds_search_cached_record_page(struct volume *volume, struct uds_request *request, + u32 chapter, u16 record_page_number, bool *found) +{ + struct cached_page *record_page; + struct index_geometry *geometry = volume->geometry; + int result; + u32 physical_page, page_number; + + *found = false; + if (record_page_number == NO_CHAPTER_INDEX_ENTRY) + return UDS_SUCCESS; + + result = VDO_ASSERT(record_page_number < geometry->record_pages_per_chapter, + "0 <= %d < %u", record_page_number, + geometry->record_pages_per_chapter); + if (result != VDO_SUCCESS) + return result; + + page_number = geometry->index_pages_per_chapter + record_page_number; + + physical_page = map_to_physical_page(volume->geometry, chapter, page_number); + + /* + * Make sure the invalidate counter is updated before we try and read the mapping. This + * prevents this thread from reading a page in the cache which has already been marked for + * invalidation by the reader thread, before the reader thread has noticed that the + * invalidate_counter has been incremented. + */ + begin_pending_search(&volume->page_cache, physical_page, request->zone_number); + + result = get_volume_page_protected(volume, request, physical_page, &record_page); + if (result != UDS_SUCCESS) { + end_pending_search(&volume->page_cache, request->zone_number); + return result; + } + + if (search_record_page(dm_bufio_get_block_data(record_page->buffer), + &request->record_name, geometry, &request->old_metadata)) + *found = true; + + end_pending_search(&volume->page_cache, request->zone_number); + return UDS_SUCCESS; +} + +void uds_prefetch_volume_chapter(const struct volume *volume, u32 chapter) +{ + const struct index_geometry *geometry = volume->geometry; + u32 physical_page = map_to_physical_page(geometry, chapter, 0); + + dm_bufio_prefetch(volume->client, physical_page, geometry->pages_per_chapter); +} + +int uds_read_chapter_index_from_volume(const struct volume *volume, u64 virtual_chapter, + struct dm_buffer *volume_buffers[], + struct delta_index_page index_pages[]) +{ + int result; + u32 i; + const struct index_geometry *geometry = volume->geometry; + u32 physical_chapter = uds_map_to_physical_chapter(geometry, virtual_chapter); + u32 physical_page = map_to_physical_page(geometry, physical_chapter, 0); + + dm_bufio_prefetch(volume->client, physical_page, geometry->index_pages_per_chapter); + for (i = 0; i < geometry->index_pages_per_chapter; i++) { + u8 *index_page; + + index_page = dm_bufio_read(volume->client, physical_page + i, + &volume_buffers[i]); + if (IS_ERR(index_page)) { + result = -PTR_ERR(index_page); + vdo_log_warning_strerror(result, + "error reading physical page %u", + physical_page); + return result; + } + + result = init_chapter_index_page(volume, index_page, physical_chapter, i, + &index_pages[i]); + if (result != UDS_SUCCESS) + return result; + } + + return UDS_SUCCESS; +} + +int uds_search_volume_page_cache(struct volume *volume, struct uds_request *request, + bool *found) +{ + int result; + u32 physical_chapter = + uds_map_to_physical_chapter(volume->geometry, request->virtual_chapter); + u32 index_page_number; + u16 record_page_number; + + index_page_number = uds_find_index_page_number(volume->index_page_map, + &request->record_name, + physical_chapter); + + if (request->location == UDS_LOCATION_INDEX_PAGE_LOOKUP) { + record_page_number = *((u16 *) &request->old_metadata); + } else { + result = search_cached_index_page(volume, request, physical_chapter, + index_page_number, + &record_page_number); + if (result != UDS_SUCCESS) + return result; + } + + return uds_search_cached_record_page(volume, request, physical_chapter, + record_page_number, found); +} + +int uds_search_volume_page_cache_for_rebuild(struct volume *volume, + const struct uds_record_name *name, + u64 virtual_chapter, bool *found) +{ + int result; + struct index_geometry *geometry = volume->geometry; + struct cached_page *page; + u32 physical_chapter = uds_map_to_physical_chapter(geometry, virtual_chapter); + u32 index_page_number; + u16 record_page_number; + u32 page_number; + + *found = false; + index_page_number = + uds_find_index_page_number(volume->index_page_map, name, + physical_chapter); + result = get_volume_page(volume, physical_chapter, index_page_number, &page); + if (result != UDS_SUCCESS) + return result; + + result = uds_search_chapter_index_page(&page->index_page, geometry, name, + &record_page_number); + if (result != UDS_SUCCESS) + return result; + + if (record_page_number == NO_CHAPTER_INDEX_ENTRY) + return UDS_SUCCESS; + + page_number = geometry->index_pages_per_chapter + record_page_number; + result = get_volume_page(volume, physical_chapter, page_number, &page); + if (result != UDS_SUCCESS) + return result; + + *found = search_record_page(dm_bufio_get_block_data(page->buffer), name, + geometry, NULL); + return UDS_SUCCESS; +} + +static void invalidate_page(struct page_cache *cache, u32 physical_page) +{ + struct cached_page *page; + int queue_index = -1; + + /* We hold the read_threads_mutex. */ + get_page_and_index(cache, physical_page, &queue_index, &page); + if (page != NULL) { + WRITE_ONCE(cache->index[page->physical_page], cache->cache_slots); + wait_for_pending_searches(cache, page->physical_page); + clear_cache_page(cache, page); + } else if (queue_index > -1) { + vdo_log_debug("setting pending read to invalid"); + cache->read_queue[queue_index].invalid = true; + } +} + +void uds_forget_chapter(struct volume *volume, u64 virtual_chapter) +{ + u32 physical_chapter = + uds_map_to_physical_chapter(volume->geometry, virtual_chapter); + u32 first_page = map_to_physical_page(volume->geometry, physical_chapter, 0); + u32 i; + + vdo_log_debug("forgetting chapter %llu", (unsigned long long) virtual_chapter); + mutex_lock(&volume->read_threads_mutex); + for (i = 0; i < volume->geometry->pages_per_chapter; i++) + invalidate_page(&volume->page_cache, first_page + i); + mutex_unlock(&volume->read_threads_mutex); +} + +/* + * Donate an index pages from a newly written chapter to the page cache since it is likely to be + * used again soon. The caller must already hold the reader thread mutex. + */ +static int donate_index_page_locked(struct volume *volume, u32 physical_chapter, + u32 index_page_number, struct dm_buffer *page_buffer) +{ + int result; + struct cached_page *page = NULL; + u32 physical_page = + map_to_physical_page(volume->geometry, physical_chapter, + index_page_number); + + page = select_victim_in_cache(&volume->page_cache); + page->buffer = page_buffer; + result = init_chapter_index_page(volume, dm_bufio_get_block_data(page_buffer), + physical_chapter, index_page_number, + &page->index_page); + if (result != UDS_SUCCESS) { + vdo_log_warning("Error initialize chapter index page"); + cancel_page_in_cache(&volume->page_cache, physical_page, page); + return result; + } + + result = put_page_in_cache(&volume->page_cache, physical_page, page); + if (result != UDS_SUCCESS) { + vdo_log_warning("Error putting page %u in cache", physical_page); + cancel_page_in_cache(&volume->page_cache, physical_page, page); + return result; + } + + return UDS_SUCCESS; +} + +static int write_index_pages(struct volume *volume, u32 physical_chapter_number, + struct open_chapter_index *chapter_index) +{ + struct index_geometry *geometry = volume->geometry; + struct dm_buffer *page_buffer; + u32 first_index_page = map_to_physical_page(geometry, physical_chapter_number, 0); + u32 delta_list_number = 0; + u32 index_page_number; + + for (index_page_number = 0; + index_page_number < geometry->index_pages_per_chapter; + index_page_number++) { + u8 *page_data; + u32 physical_page = first_index_page + index_page_number; + u32 lists_packed; + bool last_page; + int result; + + page_data = dm_bufio_new(volume->client, physical_page, &page_buffer); + if (IS_ERR(page_data)) { + return vdo_log_warning_strerror(-PTR_ERR(page_data), + "failed to prepare index page"); + } + + last_page = ((index_page_number + 1) == geometry->index_pages_per_chapter); + result = uds_pack_open_chapter_index_page(chapter_index, page_data, + delta_list_number, last_page, + &lists_packed); + if (result != UDS_SUCCESS) { + dm_bufio_release(page_buffer); + return vdo_log_warning_strerror(result, + "failed to pack index page"); + } + + dm_bufio_mark_buffer_dirty(page_buffer); + + if (lists_packed == 0) { + vdo_log_debug("no delta lists packed on chapter %u page %u", + physical_chapter_number, index_page_number); + } else { + delta_list_number += lists_packed; + } + + uds_update_index_page_map(volume->index_page_map, + chapter_index->virtual_chapter_number, + physical_chapter_number, index_page_number, + delta_list_number - 1); + + mutex_lock(&volume->read_threads_mutex); + result = donate_index_page_locked(volume, physical_chapter_number, + index_page_number, page_buffer); + mutex_unlock(&volume->read_threads_mutex); + if (result != UDS_SUCCESS) { + dm_bufio_release(page_buffer); + return result; + } + } + + return UDS_SUCCESS; +} + +static u32 encode_tree(u8 record_page[], + const struct uds_volume_record *sorted_pointers[], + u32 next_record, u32 node, u32 node_count) +{ + if (node < node_count) { + u32 child = (2 * node) + 1; + + next_record = encode_tree(record_page, sorted_pointers, next_record, + child, node_count); + + /* + * In-order traversal: copy the contents of the next record into the page at the + * node offset. + */ + memcpy(&record_page[node * BYTES_PER_RECORD], + sorted_pointers[next_record++], BYTES_PER_RECORD); + + next_record = encode_tree(record_page, sorted_pointers, next_record, + child + 1, node_count); + } + + return next_record; +} + +static int encode_record_page(const struct volume *volume, + const struct uds_volume_record records[], u8 record_page[]) +{ + int result; + u32 i; + u32 records_per_page = volume->geometry->records_per_page; + const struct uds_volume_record **record_pointers = volume->record_pointers; + + for (i = 0; i < records_per_page; i++) + record_pointers[i] = &records[i]; + + /* + * Sort the record pointers by using just the names in the records, which is less work than + * sorting the entire record values. + */ + BUILD_BUG_ON(offsetof(struct uds_volume_record, name) != 0); + result = uds_radix_sort(volume->radix_sorter, (const u8 **) record_pointers, + records_per_page, UDS_RECORD_NAME_SIZE); + if (result != UDS_SUCCESS) + return result; + + encode_tree(record_page, record_pointers, 0, 0, records_per_page); + return UDS_SUCCESS; +} + +static int write_record_pages(struct volume *volume, u32 physical_chapter_number, + const struct uds_volume_record *records) +{ + u32 record_page_number; + struct index_geometry *geometry = volume->geometry; + struct dm_buffer *page_buffer; + const struct uds_volume_record *next_record = records; + u32 first_record_page = map_to_physical_page(geometry, physical_chapter_number, + geometry->index_pages_per_chapter); + + for (record_page_number = 0; + record_page_number < geometry->record_pages_per_chapter; + record_page_number++) { + u8 *page_data; + u32 physical_page = first_record_page + record_page_number; + int result; + + page_data = dm_bufio_new(volume->client, physical_page, &page_buffer); + if (IS_ERR(page_data)) { + return vdo_log_warning_strerror(-PTR_ERR(page_data), + "failed to prepare record page"); + } + + result = encode_record_page(volume, next_record, page_data); + if (result != UDS_SUCCESS) { + dm_bufio_release(page_buffer); + return vdo_log_warning_strerror(result, + "failed to encode record page %u", + record_page_number); + } + + next_record += geometry->records_per_page; + dm_bufio_mark_buffer_dirty(page_buffer); + dm_bufio_release(page_buffer); + } + + return UDS_SUCCESS; +} + +int uds_write_chapter(struct volume *volume, struct open_chapter_index *chapter_index, + const struct uds_volume_record *records) +{ + int result; + u32 physical_chapter_number = + uds_map_to_physical_chapter(volume->geometry, + chapter_index->virtual_chapter_number); + + result = write_index_pages(volume, physical_chapter_number, chapter_index); + if (result != UDS_SUCCESS) + return result; + + result = write_record_pages(volume, physical_chapter_number, records); + if (result != UDS_SUCCESS) + return result; + + result = -dm_bufio_write_dirty_buffers(volume->client); + if (result != UDS_SUCCESS) + vdo_log_error_strerror(result, "cannot sync chapter to volume"); + + return result; +} + +static void probe_chapter(struct volume *volume, u32 chapter_number, + u64 *virtual_chapter_number) +{ + const struct index_geometry *geometry = volume->geometry; + u32 expected_list_number = 0; + u32 i; + u64 vcn = BAD_CHAPTER; + + *virtual_chapter_number = BAD_CHAPTER; + dm_bufio_prefetch(volume->client, + map_to_physical_page(geometry, chapter_number, 0), + geometry->index_pages_per_chapter); + + for (i = 0; i < geometry->index_pages_per_chapter; i++) { + struct delta_index_page *page; + int result; + + result = uds_get_volume_index_page(volume, chapter_number, i, &page); + if (result != UDS_SUCCESS) + return; + + if (page->virtual_chapter_number == BAD_CHAPTER) { + vdo_log_error("corrupt index page in chapter %u", + chapter_number); + return; + } + + if (vcn == BAD_CHAPTER) { + vcn = page->virtual_chapter_number; + } else if (page->virtual_chapter_number != vcn) { + vdo_log_error("inconsistent chapter %u index page %u: expected vcn %llu, got vcn %llu", + chapter_number, i, (unsigned long long) vcn, + (unsigned long long) page->virtual_chapter_number); + return; + } + + if (expected_list_number != page->lowest_list_number) { + vdo_log_error("inconsistent chapter %u index page %u: expected list number %u, got list number %u", + chapter_number, i, expected_list_number, + page->lowest_list_number); + return; + } + expected_list_number = page->highest_list_number + 1; + + result = uds_validate_chapter_index_page(page, geometry); + if (result != UDS_SUCCESS) + return; + } + + if (chapter_number != uds_map_to_physical_chapter(geometry, vcn)) { + vdo_log_error("chapter %u vcn %llu is out of phase (%u)", chapter_number, + (unsigned long long) vcn, geometry->chapters_per_volume); + return; + } + + *virtual_chapter_number = vcn; +} + +/* Find the last valid physical chapter in the volume. */ +static void find_real_end_of_volume(struct volume *volume, u32 limit, u32 *limit_ptr) +{ + u32 span = 1; + u32 tries = 0; + + while (limit > 0) { + u32 chapter = (span > limit) ? 0 : limit - span; + u64 vcn = 0; + + probe_chapter(volume, chapter, &vcn); + if (vcn == BAD_CHAPTER) { + limit = chapter; + if (++tries > 1) + span *= 2; + } else { + if (span == 1) + break; + span /= 2; + tries = 0; + } + } + + *limit_ptr = limit; +} + +static int find_chapter_limits(struct volume *volume, u32 chapter_limit, u64 *lowest_vcn, + u64 *highest_vcn) +{ + struct index_geometry *geometry = volume->geometry; + u64 zero_vcn; + u64 lowest = BAD_CHAPTER; + u64 highest = BAD_CHAPTER; + u64 moved_chapter = BAD_CHAPTER; + u32 left_chapter = 0; + u32 right_chapter = 0; + u32 bad_chapters = 0; + + /* + * This method assumes there is at most one run of contiguous bad chapters caused by + * unflushed writes. Either the bad spot is at the beginning and end, or somewhere in the + * middle. Wherever it is, the highest and lowest VCNs are adjacent to it. Otherwise the + * volume is cleanly saved and somewhere in the middle of it the highest VCN immediately + * precedes the lowest one. + */ + + /* It doesn't matter if this results in a bad spot (BAD_CHAPTER). */ + probe_chapter(volume, 0, &zero_vcn); + + /* + * Binary search for end of the discontinuity in the monotonically increasing virtual + * chapter numbers; bad spots are treated as a span of BAD_CHAPTER values. In effect we're + * searching for the index of the smallest value less than zero_vcn. In the case we go off + * the end it means that chapter 0 has the lowest vcn. + * + * If a virtual chapter is out-of-order, it will be the one moved by conversion. Always + * skip over the moved chapter when searching, adding it to the range at the end if + * necessary. + */ + if (geometry->remapped_physical > 0) { + u64 remapped_vcn; + + probe_chapter(volume, geometry->remapped_physical, &remapped_vcn); + if (remapped_vcn == geometry->remapped_virtual) + moved_chapter = geometry->remapped_physical; + } + + left_chapter = 0; + right_chapter = chapter_limit; + + while (left_chapter < right_chapter) { + u64 probe_vcn; + u32 chapter = (left_chapter + right_chapter) / 2; + + if (chapter == moved_chapter) + chapter--; + + probe_chapter(volume, chapter, &probe_vcn); + if (zero_vcn <= probe_vcn) { + left_chapter = chapter + 1; + if (left_chapter == moved_chapter) + left_chapter++; + } else { + right_chapter = chapter; + } + } + + /* If left_chapter goes off the end, chapter 0 has the lowest virtual chapter number.*/ + if (left_chapter >= chapter_limit) + left_chapter = 0; + + /* At this point, left_chapter is the chapter with the lowest virtual chapter number. */ + probe_chapter(volume, left_chapter, &lowest); + + /* The moved chapter might be the lowest in the range. */ + if ((moved_chapter != BAD_CHAPTER) && (lowest == geometry->remapped_virtual + 1)) + lowest = geometry->remapped_virtual; + + /* + * Circularly scan backwards, moving over any bad chapters until encountering a good one, + * which is the chapter with the highest vcn. + */ + while (highest == BAD_CHAPTER) { + right_chapter = (right_chapter + chapter_limit - 1) % chapter_limit; + if (right_chapter == moved_chapter) + continue; + + probe_chapter(volume, right_chapter, &highest); + if (bad_chapters++ >= MAX_BAD_CHAPTERS) { + vdo_log_error("too many bad chapters in volume: %u", + bad_chapters); + return UDS_CORRUPT_DATA; + } + } + + *lowest_vcn = lowest; + *highest_vcn = highest; + return UDS_SUCCESS; +} + +/* + * Find the highest and lowest contiguous chapters present in the volume and determine their + * virtual chapter numbers. This is used by rebuild. + */ +int uds_find_volume_chapter_boundaries(struct volume *volume, u64 *lowest_vcn, + u64 *highest_vcn, bool *is_empty) +{ + u32 chapter_limit = volume->geometry->chapters_per_volume; + + find_real_end_of_volume(volume, chapter_limit, &chapter_limit); + if (chapter_limit == 0) { + *lowest_vcn = 0; + *highest_vcn = 0; + *is_empty = true; + return UDS_SUCCESS; + } + + *is_empty = false; + return find_chapter_limits(volume, chapter_limit, lowest_vcn, highest_vcn); +} + +int __must_check uds_replace_volume_storage(struct volume *volume, + struct index_layout *layout, + struct block_device *bdev) +{ + int result; + u32 i; + + result = uds_replace_index_layout_storage(layout, bdev); + if (result != UDS_SUCCESS) + return result; + + /* Release all outstanding dm_bufio objects */ + for (i = 0; i < volume->page_cache.indexable_pages; i++) + volume->page_cache.index[i] = volume->page_cache.cache_slots; + for (i = 0; i < volume->page_cache.cache_slots; i++) + clear_cache_page(&volume->page_cache, &volume->page_cache.cache[i]); + if (volume->sparse_cache != NULL) + uds_invalidate_sparse_cache(volume->sparse_cache); + if (volume->client != NULL) + dm_bufio_client_destroy(vdo_forget(volume->client)); + + return uds_open_volume_bufio(layout, volume->geometry->bytes_per_page, + volume->reserved_buffers, &volume->client); +} + +static int __must_check initialize_page_cache(struct page_cache *cache, + const struct index_geometry *geometry, + u32 chapters_in_cache, + unsigned int zone_count) +{ + int result; + u32 i; + + cache->indexable_pages = geometry->pages_per_volume + 1; + cache->cache_slots = chapters_in_cache * geometry->record_pages_per_chapter; + cache->zone_count = zone_count; + atomic64_set(&cache->clock, 1); + + result = VDO_ASSERT((cache->cache_slots <= VOLUME_CACHE_MAX_ENTRIES), + "requested cache size, %u, within limit %u", + cache->cache_slots, VOLUME_CACHE_MAX_ENTRIES); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate(VOLUME_CACHE_MAX_QUEUED_READS, struct queued_read, + "volume read queue", &cache->read_queue); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate(cache->zone_count, struct search_pending_counter, + "Volume Cache Zones", &cache->search_pending_counters); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate(cache->indexable_pages, u16, "page cache index", + &cache->index); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate(cache->cache_slots, struct cached_page, "page cache cache", + &cache->cache); + if (result != VDO_SUCCESS) + return result; + + /* Initialize index values to invalid values. */ + for (i = 0; i < cache->indexable_pages; i++) + cache->index[i] = cache->cache_slots; + + for (i = 0; i < cache->cache_slots; i++) + clear_cache_page(cache, &cache->cache[i]); + + return UDS_SUCCESS; +} + +int uds_make_volume(const struct uds_configuration *config, struct index_layout *layout, + struct volume **new_volume) +{ + unsigned int i; + struct volume *volume = NULL; + struct index_geometry *geometry; + unsigned int reserved_buffers; + int result; + + result = vdo_allocate(1, struct volume, "volume", &volume); + if (result != VDO_SUCCESS) + return result; + + volume->nonce = uds_get_volume_nonce(layout); + + result = uds_copy_index_geometry(config->geometry, &volume->geometry); + if (result != UDS_SUCCESS) { + uds_free_volume(volume); + return vdo_log_warning_strerror(result, + "failed to allocate geometry: error"); + } + geometry = volume->geometry; + + /* + * Reserve a buffer for each entry in the page cache, one for the chapter writer, and one + * for each entry in the sparse cache. + */ + reserved_buffers = config->cache_chapters * geometry->record_pages_per_chapter; + reserved_buffers += 1; + if (uds_is_sparse_index_geometry(geometry)) + reserved_buffers += (config->cache_chapters * geometry->index_pages_per_chapter); + volume->reserved_buffers = reserved_buffers; + result = uds_open_volume_bufio(layout, geometry->bytes_per_page, + volume->reserved_buffers, &volume->client); + if (result != UDS_SUCCESS) { + uds_free_volume(volume); + return result; + } + + result = uds_make_radix_sorter(geometry->records_per_page, + &volume->radix_sorter); + if (result != UDS_SUCCESS) { + uds_free_volume(volume); + return result; + } + + result = vdo_allocate(geometry->records_per_page, + const struct uds_volume_record *, "record pointers", + &volume->record_pointers); + if (result != VDO_SUCCESS) { + uds_free_volume(volume); + return result; + } + + if (uds_is_sparse_index_geometry(geometry)) { + size_t page_size = sizeof(struct delta_index_page) + geometry->bytes_per_page; + + result = uds_make_sparse_cache(geometry, config->cache_chapters, + config->zone_count, + &volume->sparse_cache); + if (result != UDS_SUCCESS) { + uds_free_volume(volume); + return result; + } + + volume->cache_size = + page_size * geometry->index_pages_per_chapter * config->cache_chapters; + } + + result = initialize_page_cache(&volume->page_cache, geometry, + config->cache_chapters, config->zone_count); + if (result != UDS_SUCCESS) { + uds_free_volume(volume); + return result; + } + + volume->cache_size += volume->page_cache.cache_slots * sizeof(struct delta_index_page); + result = uds_make_index_page_map(geometry, &volume->index_page_map); + if (result != UDS_SUCCESS) { + uds_free_volume(volume); + return result; + } + + mutex_init(&volume->read_threads_mutex); + uds_init_cond(&volume->read_threads_read_done_cond); + uds_init_cond(&volume->read_threads_cond); + + result = vdo_allocate(config->read_threads, struct thread *, "reader threads", + &volume->reader_threads); + if (result != VDO_SUCCESS) { + uds_free_volume(volume); + return result; + } + + for (i = 0; i < config->read_threads; i++) { + result = vdo_create_thread(read_thread_function, (void *) volume, + "reader", &volume->reader_threads[i]); + if (result != VDO_SUCCESS) { + uds_free_volume(volume); + return result; + } + + volume->read_thread_count = i + 1; + } + + *new_volume = volume; + return UDS_SUCCESS; +} + +static void uninitialize_page_cache(struct page_cache *cache) +{ + u16 i; + + if (cache->cache != NULL) { + for (i = 0; i < cache->cache_slots; i++) + release_page_buffer(&cache->cache[i]); + } + vdo_free(cache->index); + vdo_free(cache->cache); + vdo_free(cache->search_pending_counters); + vdo_free(cache->read_queue); +} + +void uds_free_volume(struct volume *volume) +{ + if (volume == NULL) + return; + + if (volume->reader_threads != NULL) { + unsigned int i; + + /* This works even if some threads weren't started. */ + mutex_lock(&volume->read_threads_mutex); + volume->read_threads_exiting = true; + uds_broadcast_cond(&volume->read_threads_cond); + mutex_unlock(&volume->read_threads_mutex); + for (i = 0; i < volume->read_thread_count; i++) + vdo_join_threads(volume->reader_threads[i]); + vdo_free(volume->reader_threads); + volume->reader_threads = NULL; + } + + /* Must destroy the client AFTER freeing the cached pages. */ + uninitialize_page_cache(&volume->page_cache); + uds_free_sparse_cache(volume->sparse_cache); + if (volume->client != NULL) + dm_bufio_client_destroy(vdo_forget(volume->client)); + + uds_free_index_page_map(volume->index_page_map); + uds_free_radix_sorter(volume->radix_sorter); + vdo_free(volume->geometry); + vdo_free(volume->record_pointers); + vdo_free(volume); +} diff --git a/drivers/md/dm-vdo/indexer/volume.h b/drivers/md/dm-vdo/indexer/volume.h new file mode 100644 index 000000000000..8679a5e55347 --- /dev/null +++ b/drivers/md/dm-vdo/indexer/volume.h @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_VOLUME_H +#define UDS_VOLUME_H + +#include +#include +#include +#include + +#include "permassert.h" +#include "thread-utils.h" + +#include "chapter-index.h" +#include "config.h" +#include "geometry.h" +#include "indexer.h" +#include "index-layout.h" +#include "index-page-map.h" +#include "radix-sort.h" +#include "sparse-cache.h" + +/* + * The volume manages deduplication records on permanent storage. The term "volume" can also refer + * to the region of permanent storage where the records (and the chapters containing them) are + * stored. The volume handles all I/O to this region by reading, caching, and writing chapter pages + * as necessary. + */ + +enum index_lookup_mode { + /* Always do lookups in all chapters normally */ + LOOKUP_NORMAL, + /* Only do a subset of lookups needed when rebuilding an index */ + LOOKUP_FOR_REBUILD, +}; + +struct queued_read { + bool invalid; + bool reserved; + u32 physical_page; + struct uds_request *first_request; + struct uds_request *last_request; +}; + +struct __aligned(L1_CACHE_BYTES) search_pending_counter { + u64 atomic_value; +}; + +struct cached_page { + /* Whether this page is currently being read asynchronously */ + bool read_pending; + /* The physical page stored in this cache entry */ + u32 physical_page; + /* The value of the volume clock when this page was last used */ + s64 last_used; + /* The cached page buffer */ + struct dm_buffer *buffer; + /* The chapter index page, meaningless for record pages */ + struct delta_index_page index_page; +}; + +struct page_cache { + /* The number of zones */ + unsigned int zone_count; + /* The number of volume pages that can be cached */ + u32 indexable_pages; + /* The maximum number of simultaneously cached pages */ + u16 cache_slots; + /* An index for each physical page noting where it is in the cache */ + u16 *index; + /* The array of cached pages */ + struct cached_page *cache; + /* A counter for each zone tracking if a search is occurring there */ + struct search_pending_counter *search_pending_counters; + /* The read queue entries as a circular array */ + struct queued_read *read_queue; + + /* All entries above this point are constant after initialization. */ + + /* + * These values are all indexes into the array of read queue entries. New entries in the + * read queue are enqueued at read_queue_last. To dequeue entries, a reader thread gets the + * lock and then claims the entry pointed to by read_queue_next_read and increments that + * value. After the read is completed, the reader thread calls release_read_queue_entry(), + * which increments read_queue_first until it points to a pending read, or is equal to + * read_queue_next_read. This means that if multiple reads are outstanding, + * read_queue_first might not advance until the last of the reads finishes. + */ + u16 read_queue_first; + u16 read_queue_next_read; + u16 read_queue_last; + + atomic64_t clock; +}; + +struct volume { + struct index_geometry *geometry; + struct dm_bufio_client *client; + u64 nonce; + size_t cache_size; + + /* A single page worth of records, for sorting */ + const struct uds_volume_record **record_pointers; + /* Sorter for sorting records within each page */ + struct radix_sorter *radix_sorter; + + struct sparse_cache *sparse_cache; + struct page_cache page_cache; + struct index_page_map *index_page_map; + + struct mutex read_threads_mutex; + struct cond_var read_threads_cond; + struct cond_var read_threads_read_done_cond; + struct thread **reader_threads; + unsigned int read_thread_count; + bool read_threads_exiting; + + enum index_lookup_mode lookup_mode; + unsigned int reserved_buffers; +}; + +int __must_check uds_make_volume(const struct uds_configuration *config, + struct index_layout *layout, + struct volume **new_volume); + +void uds_free_volume(struct volume *volume); + +int __must_check uds_replace_volume_storage(struct volume *volume, + struct index_layout *layout, + struct block_device *bdev); + +int __must_check uds_find_volume_chapter_boundaries(struct volume *volume, + u64 *lowest_vcn, u64 *highest_vcn, + bool *is_empty); + +int __must_check uds_search_volume_page_cache(struct volume *volume, + struct uds_request *request, + bool *found); + +int __must_check uds_search_volume_page_cache_for_rebuild(struct volume *volume, + const struct uds_record_name *name, + u64 virtual_chapter, + bool *found); + +int __must_check uds_search_cached_record_page(struct volume *volume, + struct uds_request *request, u32 chapter, + u16 record_page_number, bool *found); + +void uds_forget_chapter(struct volume *volume, u64 chapter); + +int __must_check uds_write_chapter(struct volume *volume, + struct open_chapter_index *chapter_index, + const struct uds_volume_record records[]); + +void uds_prefetch_volume_chapter(const struct volume *volume, u32 chapter); + +int __must_check uds_read_chapter_index_from_volume(const struct volume *volume, + u64 virtual_chapter, + struct dm_buffer *volume_buffers[], + struct delta_index_page index_pages[]); + +int __must_check uds_get_volume_record_page(struct volume *volume, u32 chapter, + u32 page_number, u8 **data_ptr); + +int __must_check uds_get_volume_index_page(struct volume *volume, u32 chapter, + u32 page_number, + struct delta_index_page **page_ptr); + +#endif /* UDS_VOLUME_H */ diff --git a/drivers/md/dm-vdo/int-map.c b/drivers/md/dm-vdo/int-map.c new file mode 100644 index 000000000000..3aa438f84ea1 --- /dev/null +++ b/drivers/md/dm-vdo/int-map.c @@ -0,0 +1,707 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +/** + * DOC: + * + * Hash table implementation of a map from integers to pointers, implemented using the Hopscotch + * Hashing algorithm by Herlihy, Shavit, and Tzafrir (see + * http://en.wikipedia.org/wiki/Hopscotch_hashing). This implementation does not contain any of the + * locking/concurrency features of the algorithm, just the collision resolution scheme. + * + * Hopscotch Hashing is based on hashing with open addressing and linear probing. All the entries + * are stored in a fixed array of buckets, with no dynamic allocation for collisions. Unlike linear + * probing, all the entries that hash to a given bucket are stored within a fixed neighborhood + * starting at that bucket. Chaining is effectively represented as a bit vector relative to each + * bucket instead of as pointers or explicit offsets. + * + * When an empty bucket cannot be found within a given neighborhood, subsequent neighborhoods are + * searched, and one or more entries will "hop" into those neighborhoods. When this process works, + * an empty bucket will move into the desired neighborhood, allowing the entry to be added. When + * that process fails (typically when the buckets are around 90% full), the table must be resized + * and the all entries rehashed and added to the expanded table. + * + * Unlike linear probing, the number of buckets that must be searched in the worst case has a fixed + * upper bound (the size of the neighborhood). Those entries occupy a small number of memory cache + * lines, leading to improved use of the cache (fewer misses on both successful and unsuccessful + * searches). Hopscotch hashing outperforms linear probing at much higher load factors, so even + * with the increased memory burden for maintaining the hop vectors, less memory is needed to + * achieve that performance. Hopscotch is also immune to "contamination" from deleting entries + * since entries are genuinely removed instead of being replaced by a placeholder. + * + * The published description of the algorithm used a bit vector, but the paper alludes to an offset + * scheme which is used by this implementation. Since the entries in the neighborhood are within N + * entries of the hash bucket at the start of the neighborhood, a pair of small offset fields each + * log2(N) bits wide is all that's needed to maintain the hops as a linked list. In order to encode + * "no next hop" (i.e. NULL) as the natural initial value of zero, the offsets are biased by one + * (i.e. 0 => NULL, 1 => offset=0, 2 => offset=1, etc.) We can represent neighborhoods of up to 255 + * entries with just 8+8=16 bits per entry. The hop list is sorted by hop offset so the first entry + * in the list is always the bucket closest to the start of the neighborhood. + * + * While individual accesses tend to be very fast, the table resize operations are very, very + * expensive. If an upper bound on the latency of adding an entry to the table is needed, we either + * need to ensure the table is pre-sized to be large enough so no resize is ever needed, or we'll + * need to develop an approach to incrementally resize the table. + */ + +#include "int-map.h" + +#include + +#include "errors.h" +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "permassert.h" + +#define DEFAULT_CAPACITY 16 /* the number of neighborhoods in a new table */ +#define NEIGHBORHOOD 255 /* the number of buckets in each neighborhood */ +#define MAX_PROBES 1024 /* limit on the number of probes for a free bucket */ +#define NULL_HOP_OFFSET 0 /* the hop offset value terminating the hop list */ +#define DEFAULT_LOAD 75 /* a compromise between memory use and performance */ + +/** + * struct bucket - hash bucket + * + * Buckets are packed together to reduce memory usage and improve cache efficiency. It would be + * tempting to encode the hop offsets separately and maintain alignment of key/value pairs, but + * it's crucial to keep the hop fields near the buckets that they use them so they'll tend to share + * cache lines. + */ +struct __packed bucket { + /** + * @first_hop: The biased offset of the first entry in the hop list of the neighborhood + * that hashes to this bucket. + */ + u8 first_hop; + /** @next_hop: The biased offset of the next bucket in the hop list. */ + u8 next_hop; + /** @key: The key stored in this bucket. */ + u64 key; + /** @value: The value stored in this bucket (NULL if empty). */ + void *value; +}; + +/** + * struct int_map - The concrete definition of the opaque int_map type. + * + * To avoid having to wrap the neighborhoods of the last entries back around to the start of the + * bucket array, we allocate a few more buckets at the end of the array instead, which is why + * capacity and bucket_count are different. + */ +struct int_map { + /** @size: The number of entries stored in the map. */ + size_t size; + /** @capacity: The number of neighborhoods in the map. */ + size_t capacity; + /* @bucket_count: The number of buckets in the bucket array. */ + size_t bucket_count; + /** @buckets: The array of hash buckets. */ + struct bucket *buckets; +}; + +/** + * mix() - The Google CityHash 16-byte hash mixing function. + * @input1: The first input value. + * @input2: The second input value. + * + * Return: A hash of the two inputs. + */ +static u64 mix(u64 input1, u64 input2) +{ + static const u64 CITY_MULTIPLIER = 0x9ddfea08eb382d69ULL; + u64 hash = (input1 ^ input2); + + hash *= CITY_MULTIPLIER; + hash ^= (hash >> 47); + hash ^= input2; + hash *= CITY_MULTIPLIER; + hash ^= (hash >> 47); + hash *= CITY_MULTIPLIER; + return hash; +} + +/** + * hash_key() - Calculate a 64-bit non-cryptographic hash value for the provided 64-bit integer + * key. + * @key: The mapping key. + * + * The implementation is based on Google's CityHash, only handling the specific case of an 8-byte + * input. + * + * Return: The hash of the mapping key. + */ +static u64 hash_key(u64 key) +{ + /* + * Aliasing restrictions forbid us from casting pointer types, so use a union to convert a + * single u64 to two u32 values. + */ + union { + u64 u64; + u32 u32[2]; + } pun = {.u64 = key}; + + return mix(sizeof(key) + (((u64) pun.u32[0]) << 3), pun.u32[1]); +} + +/** + * allocate_buckets() - Initialize an int_map. + * @map: The map to initialize. + * @capacity: The initial capacity of the map. + * + * Return: VDO_SUCCESS or an error code. + */ +static int allocate_buckets(struct int_map *map, size_t capacity) +{ + map->size = 0; + map->capacity = capacity; + + /* + * Allocate NEIGHBORHOOD - 1 extra buckets so the last bucket can have a full neighborhood + * without have to wrap back around to element zero. + */ + map->bucket_count = capacity + (NEIGHBORHOOD - 1); + return vdo_allocate(map->bucket_count, struct bucket, + "struct int_map buckets", &map->buckets); +} + +/** + * vdo_int_map_create() - Allocate and initialize an int_map. + * @initial_capacity: The number of entries the map should initially be capable of holding (zero + * tells the map to use its own small default). + * @map_ptr: Output, a pointer to hold the new int_map. + * + * Return: VDO_SUCCESS or an error code. + */ +int vdo_int_map_create(size_t initial_capacity, struct int_map **map_ptr) +{ + struct int_map *map; + int result; + size_t capacity; + + result = vdo_allocate(1, struct int_map, "struct int_map", &map); + if (result != VDO_SUCCESS) + return result; + + /* Use the default capacity if the caller did not specify one. */ + capacity = (initial_capacity > 0) ? initial_capacity : DEFAULT_CAPACITY; + + /* + * Scale up the capacity by the specified initial load factor. (i.e to hold 1000 entries at + * 80% load we need a capacity of 1250) + */ + capacity = capacity * 100 / DEFAULT_LOAD; + + result = allocate_buckets(map, capacity); + if (result != VDO_SUCCESS) { + vdo_int_map_free(vdo_forget(map)); + return result; + } + + *map_ptr = map; + return VDO_SUCCESS; +} + +/** + * vdo_int_map_free() - Free an int_map. + * @map: The int_map to free. + * + * NOTE: The map does not own the pointer values stored in the map and they are not freed by this + * call. + */ +void vdo_int_map_free(struct int_map *map) +{ + if (map == NULL) + return; + + vdo_free(vdo_forget(map->buckets)); + vdo_free(vdo_forget(map)); +} + +/** + * vdo_int_map_size() - Get the number of entries stored in an int_map. + * @map: The int_map to query. + * + * Return: The number of entries in the map. + */ +size_t vdo_int_map_size(const struct int_map *map) +{ + return map->size; +} + +/** + * dereference_hop() - Convert a biased hop offset within a neighborhood to a pointer to the bucket + * it references. + * @neighborhood: The first bucket in the neighborhood. + * @hop_offset: The biased hop offset to the desired bucket. + * + * Return: NULL if hop_offset is zero, otherwise a pointer to the bucket in the neighborhood at + * hop_offset - 1. + */ +static struct bucket *dereference_hop(struct bucket *neighborhood, unsigned int hop_offset) +{ + BUILD_BUG_ON(NULL_HOP_OFFSET != 0); + if (hop_offset == NULL_HOP_OFFSET) + return NULL; + + return &neighborhood[hop_offset - 1]; +} + +/** + * insert_in_hop_list() - Add a bucket into the hop list for the neighborhood. + * @neighborhood: The first bucket in the neighborhood. + * @new_bucket: The bucket to add to the hop list. + * + * The bucket is inserted it into the list so the hop list remains sorted by hop offset. + */ +static void insert_in_hop_list(struct bucket *neighborhood, struct bucket *new_bucket) +{ + /* Zero indicates a NULL hop offset, so bias the hop offset by one. */ + int hop_offset = 1 + (new_bucket - neighborhood); + + /* Handle the special case of adding a bucket at the start of the list. */ + int next_hop = neighborhood->first_hop; + + if ((next_hop == NULL_HOP_OFFSET) || (next_hop > hop_offset)) { + new_bucket->next_hop = next_hop; + neighborhood->first_hop = hop_offset; + return; + } + + /* Search the hop list for the insertion point that maintains the sort order. */ + for (;;) { + struct bucket *bucket = dereference_hop(neighborhood, next_hop); + + next_hop = bucket->next_hop; + + if ((next_hop == NULL_HOP_OFFSET) || (next_hop > hop_offset)) { + new_bucket->next_hop = next_hop; + bucket->next_hop = hop_offset; + return; + } + } +} + +/** + * select_bucket() - Select and return the hash bucket for a given search key. + * @map: The map to search. + * @key: The mapping key. + */ +static struct bucket *select_bucket(const struct int_map *map, u64 key) +{ + /* + * Calculate a good hash value for the provided key. We want exactly 32 bits, so mask the + * result. + */ + u64 hash = hash_key(key) & 0xFFFFFFFF; + + /* + * Scale the 32-bit hash to a bucket index by treating it as a binary fraction and + * multiplying that by the capacity. If the hash is uniformly distributed over [0 .. + * 2^32-1], then (hash * capacity / 2^32) should be uniformly distributed over [0 .. + * capacity-1]. The multiply and shift is much faster than a divide (modulus) on X86 CPUs. + */ + return &map->buckets[(hash * map->capacity) >> 32]; +} + +/** + * search_hop_list() - Search the hop list associated with given hash bucket for a given search + * key. + * @map: The map being searched. + * @bucket: The map bucket to search for the key. + * @key: The mapping key. + * @previous_ptr: Output. if not NULL, a pointer in which to store the bucket in the list preceding + * the one that had the matching key + * + * If the key is found, returns a pointer to the entry (bucket or collision), otherwise returns + * NULL. + * + * Return: An entry that matches the key, or NULL if not found. + */ +static struct bucket *search_hop_list(struct int_map *map __always_unused, + struct bucket *bucket, + u64 key, + struct bucket **previous_ptr) +{ + struct bucket *previous = NULL; + unsigned int next_hop = bucket->first_hop; + + while (next_hop != NULL_HOP_OFFSET) { + /* + * Check the neighboring bucket indexed by the offset for the + * desired key. + */ + struct bucket *entry = dereference_hop(bucket, next_hop); + + if ((key == entry->key) && (entry->value != NULL)) { + if (previous_ptr != NULL) + *previous_ptr = previous; + return entry; + } + next_hop = entry->next_hop; + previous = entry; + } + + return NULL; +} + +/** + * vdo_int_map_get() - Retrieve the value associated with a given key from the int_map. + * @map: The int_map to query. + * @key: The key to look up. + * + * Return: The value associated with the given key, or NULL if the key is not mapped to any value. + */ +void *vdo_int_map_get(struct int_map *map, u64 key) +{ + struct bucket *match = search_hop_list(map, select_bucket(map, key), key, NULL); + + return ((match != NULL) ? match->value : NULL); +} + +/** + * resize_buckets() - Increase the number of hash buckets. + * @map: The map to resize. + * + * Resizes and rehashes all the existing entries, storing them in the new buckets. + * + * Return: VDO_SUCCESS or an error code. + */ +static int resize_buckets(struct int_map *map) +{ + int result; + size_t i; + + /* Copy the top-level map data to the stack. */ + struct int_map old_map = *map; + + /* Re-initialize the map to be empty and 50% larger. */ + size_t new_capacity = map->capacity / 2 * 3; + + vdo_log_info("%s: attempting resize from %zu to %zu, current size=%zu", + __func__, map->capacity, new_capacity, map->size); + result = allocate_buckets(map, new_capacity); + if (result != VDO_SUCCESS) { + *map = old_map; + return result; + } + + /* Populate the new hash table from the entries in the old bucket array. */ + for (i = 0; i < old_map.bucket_count; i++) { + struct bucket *entry = &old_map.buckets[i]; + + if (entry->value == NULL) + continue; + + result = vdo_int_map_put(map, entry->key, entry->value, true, NULL); + if (result != VDO_SUCCESS) { + /* Destroy the new partial map and restore the map from the stack. */ + vdo_free(vdo_forget(map->buckets)); + *map = old_map; + return result; + } + } + + /* Destroy the old bucket array. */ + vdo_free(vdo_forget(old_map.buckets)); + return VDO_SUCCESS; +} + +/** + * find_empty_bucket() - Probe the bucket array starting at the given bucket for the next empty + * bucket, returning a pointer to it. + * @map: The map containing the buckets to search. + * @bucket: The bucket at which to start probing. + * @max_probes: The maximum number of buckets to search. + * + * NULL will be returned if the search reaches the end of the bucket array or if the number of + * linear probes exceeds a specified limit. + * + * Return: The next empty bucket, or NULL if the search failed. + */ +static struct bucket * +find_empty_bucket(struct int_map *map, struct bucket *bucket, unsigned int max_probes) +{ + /* + * Limit the search to either the nearer of the end of the bucket array or a fixed distance + * beyond the initial bucket. + */ + ptrdiff_t remaining = &map->buckets[map->bucket_count] - bucket; + struct bucket *sentinel = &bucket[min_t(ptrdiff_t, remaining, max_probes)]; + struct bucket *entry; + + for (entry = bucket; entry < sentinel; entry++) { + if (entry->value == NULL) + return entry; + } + + return NULL; +} + +/** + * move_empty_bucket() - Move an empty bucket closer to the start of the bucket array. + * @map: The map containing the bucket. + * @hole: The empty bucket to fill with an entry that precedes it in one of its enclosing + * neighborhoods. + * + * This searches the neighborhoods that contain the empty bucket for a non-empty bucket closer to + * the start of the array. If such a bucket is found, this swaps the two buckets by moving the + * entry to the empty bucket. + * + * Return: The bucket that was vacated by moving its entry to the provided hole, or NULL if no + * entry could be moved. + */ +static struct bucket *move_empty_bucket(struct int_map *map __always_unused, + struct bucket *hole) +{ + /* + * Examine every neighborhood that the empty bucket is part of, starting with the one in + * which it is the last bucket. No boundary check is needed for the negative array + * arithmetic since this function is only called when hole is at least NEIGHBORHOOD cells + * deeper into the array than a valid bucket. + */ + struct bucket *bucket; + + for (bucket = &hole[1 - NEIGHBORHOOD]; bucket < hole; bucket++) { + /* + * Find the entry that is nearest to the bucket, which means it will be nearest to + * the hash bucket whose neighborhood is full. + */ + struct bucket *new_hole = dereference_hop(bucket, bucket->first_hop); + + if (new_hole == NULL) { + /* + * There are no buckets in this neighborhood that are in use by this one + * (they must all be owned by overlapping neighborhoods). + */ + continue; + } + + /* + * Skip this bucket if its first entry is actually further away than the hole that + * we're already trying to fill. + */ + if (hole < new_hole) + continue; + + /* + * We've found an entry in this neighborhood that we can "hop" further away, moving + * the hole closer to the hash bucket, if not all the way into its neighborhood. + */ + + /* + * The entry that will be the new hole is the first bucket in the list, so setting + * first_hop is all that's needed remove it from the list. + */ + bucket->first_hop = new_hole->next_hop; + new_hole->next_hop = NULL_HOP_OFFSET; + + /* Move the entry into the original hole. */ + hole->key = new_hole->key; + hole->value = new_hole->value; + new_hole->value = NULL; + + /* Insert the filled hole into the hop list for the neighborhood. */ + insert_in_hop_list(bucket, hole); + return new_hole; + } + + /* We couldn't find an entry to relocate to the hole. */ + return NULL; +} + +/** + * update_mapping() - Find and update any existing mapping for a given key, returning the value + * associated with the key in the provided pointer. + * @map: The int_map to attempt to modify. + * @neighborhood: The first bucket in the neighborhood that would contain the search key + * @key: The key with which to associate the new value. + * @new_value: The value to be associated with the key. + * @update: Whether to overwrite an existing value. + * @old_value_ptr: a pointer in which to store the old value (unmodified if no mapping was found) + * + * Return: true if the map contains a mapping for the key, false if it does not. + */ +static bool update_mapping(struct int_map *map, struct bucket *neighborhood, + u64 key, void *new_value, bool update, void **old_value_ptr) +{ + struct bucket *bucket = search_hop_list(map, neighborhood, key, NULL); + + if (bucket == NULL) { + /* There is no bucket containing the key in the neighborhood. */ + return false; + } + + /* + * Return the value of the current mapping (if desired) and update the mapping with the new + * value (if desired). + */ + if (old_value_ptr != NULL) + *old_value_ptr = bucket->value; + if (update) + bucket->value = new_value; + return true; +} + +/** + * find_or_make_vacancy() - Find an empty bucket. + * @map: The int_map to search or modify. + * @neighborhood: The first bucket in the neighborhood in which an empty bucket is needed for a new + * mapping. + * + * Find an empty bucket in a specified neighborhood for a new mapping or attempt to re-arrange + * mappings so there is such a bucket. This operation may fail (returning NULL) if an empty bucket + * is not available or could not be relocated to the neighborhood. + * + * Return: a pointer to an empty bucket in the desired neighborhood, or NULL if a vacancy could not + * be found or arranged. + */ +static struct bucket *find_or_make_vacancy(struct int_map *map, + struct bucket *neighborhood) +{ + /* Probe within and beyond the neighborhood for the first empty bucket. */ + struct bucket *hole = find_empty_bucket(map, neighborhood, MAX_PROBES); + + /* + * Keep trying until the empty bucket is in the bucket's neighborhood or we are unable to + * move it any closer by swapping it with a filled bucket. + */ + while (hole != NULL) { + int distance = hole - neighborhood; + + if (distance < NEIGHBORHOOD) { + /* + * We've found or relocated an empty bucket close enough to the initial + * hash bucket to be referenced by its hop vector. + */ + return hole; + } + + /* + * The nearest empty bucket isn't within the neighborhood that must contain the new + * entry, so try to swap it with bucket that is closer. + */ + hole = move_empty_bucket(map, hole); + } + + return NULL; +} + +/** + * vdo_int_map_put() - Try to associate a value with an integer. + * @map: The int_map to attempt to modify. + * @key: The key with which to associate the new value. + * @new_value: The value to be associated with the key. + * @update: Whether to overwrite an existing value. + * @old_value_ptr: A pointer in which to store either the old value (if the key was already mapped) + * or NULL if the map did not contain the key; NULL may be provided if the caller + * does not need to know the old value + * + * Try to associate a value (a pointer) with an integer in an int_map. If the map already contains + * a mapping for the provided key, the old value is only replaced with the specified value if + * update is true. In either case the old value is returned. If the map does not already contain a + * value for the specified key, the new value is added regardless of the value of update. + * + * Return: VDO_SUCCESS or an error code. + */ +int vdo_int_map_put(struct int_map *map, u64 key, void *new_value, bool update, + void **old_value_ptr) +{ + struct bucket *neighborhood, *bucket; + + if (unlikely(new_value == NULL)) + return -EINVAL; + + /* + * Select the bucket at the start of the neighborhood that must contain any entry for the + * provided key. + */ + neighborhood = select_bucket(map, key); + + /* + * Check whether the neighborhood already contains an entry for the key, in which case we + * optionally update it, returning the old value. + */ + if (update_mapping(map, neighborhood, key, new_value, update, old_value_ptr)) + return VDO_SUCCESS; + + /* + * Find an empty bucket in the desired neighborhood for the new entry or re-arrange entries + * in the map so there is such a bucket. This operation will usually succeed; the loop body + * will only be executed on the rare occasions that we have to resize the map. + */ + while ((bucket = find_or_make_vacancy(map, neighborhood)) == NULL) { + int result; + + /* + * There is no empty bucket in which to put the new entry in the current map, so + * we're forced to allocate a new bucket array with a larger capacity, re-hash all + * the entries into those buckets, and try again (a very expensive operation for + * large maps). + */ + result = resize_buckets(map); + if (result != VDO_SUCCESS) + return result; + + /* + * Resizing the map invalidates all pointers to buckets, so recalculate the + * neighborhood pointer. + */ + neighborhood = select_bucket(map, key); + } + + /* Put the new entry in the empty bucket, adding it to the neighborhood. */ + bucket->key = key; + bucket->value = new_value; + insert_in_hop_list(neighborhood, bucket); + map->size += 1; + + /* There was no existing entry, so there was no old value to be returned. */ + if (old_value_ptr != NULL) + *old_value_ptr = NULL; + return VDO_SUCCESS; +} + +/** + * vdo_int_map_remove() - Remove the mapping for a given key from the int_map. + * @map: The int_map from which to remove the mapping. + * @key: The key whose mapping is to be removed. + * + * Return: the value that was associated with the key, or NULL if it was not mapped. + */ +void *vdo_int_map_remove(struct int_map *map, u64 key) +{ + void *value; + + /* Select the bucket to search and search it for an existing entry. */ + struct bucket *bucket = select_bucket(map, key); + struct bucket *previous; + struct bucket *victim = search_hop_list(map, bucket, key, &previous); + + if (victim == NULL) { + /* There is no matching entry to remove. */ + return NULL; + } + + /* + * We found an entry to remove. Save the mapped value to return later and empty the bucket. + */ + map->size -= 1; + value = victim->value; + victim->value = NULL; + victim->key = 0; + + /* The victim bucket is now empty, but it still needs to be spliced out of the hop list. */ + if (previous == NULL) { + /* The victim is the head of the list, so swing first_hop. */ + bucket->first_hop = victim->next_hop; + } else { + previous->next_hop = victim->next_hop; + } + + victim->next_hop = NULL_HOP_OFFSET; + return value; +} diff --git a/drivers/md/dm-vdo/int-map.h b/drivers/md/dm-vdo/int-map.h new file mode 100644 index 000000000000..1858ad799887 --- /dev/null +++ b/drivers/md/dm-vdo/int-map.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_INT_MAP_H +#define VDO_INT_MAP_H + +#include +#include + +/** + * DOC: int_map + * + * An int_map associates pointers (void *) with integer keys (u64). NULL pointer values are + * not supported. + * + * The map is implemented as hash table, which should provide constant-time insert, query, and + * remove operations, although the insert may occasionally grow the table, which is linear in the + * number of entries in the map. The table will grow as needed to hold new entries, but will not + * shrink as entries are removed. + */ + +struct int_map; + +int __must_check vdo_int_map_create(size_t initial_capacity, struct int_map **map_ptr); + +void vdo_int_map_free(struct int_map *map); + +size_t vdo_int_map_size(const struct int_map *map); + +void *vdo_int_map_get(struct int_map *map, u64 key); + +int __must_check vdo_int_map_put(struct int_map *map, u64 key, void *new_value, + bool update, void **old_value_ptr); + +void *vdo_int_map_remove(struct int_map *map, u64 key); + +#endif /* VDO_INT_MAP_H */ diff --git a/drivers/md/dm-vdo/io-submitter.c b/drivers/md/dm-vdo/io-submitter.c new file mode 100644 index 000000000000..9a3716bb3c05 --- /dev/null +++ b/drivers/md/dm-vdo/io-submitter.c @@ -0,0 +1,477 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "io-submitter.h" + +#include +#include +#include + +#include "memory-alloc.h" +#include "permassert.h" + +#include "data-vio.h" +#include "logger.h" +#include "types.h" +#include "vdo.h" +#include "vio.h" + +/* + * Submission of bio operations to the underlying storage device will go through a separate work + * queue thread (or more than one) to prevent blocking in other threads if the storage device has a + * full queue. The plug structure allows that thread to do better batching of requests to make the + * I/O more efficient. + * + * When multiple worker threads are used, a thread is chosen for a I/O operation submission based + * on the PBN, so a given PBN will consistently wind up on the same thread. Flush operations are + * assigned round-robin. + * + * The map (protected by the mutex) collects pending I/O operations so that the worker thread can + * reorder them to try to encourage I/O request merging in the request queue underneath. + */ +struct bio_queue_data { + struct vdo_work_queue *queue; + struct blk_plug plug; + struct int_map *map; + struct mutex lock; + unsigned int queue_number; +}; + +struct io_submitter { + unsigned int num_bio_queues_used; + unsigned int bio_queue_rotation_interval; + struct bio_queue_data bio_queue_data[]; +}; + +static void start_bio_queue(void *ptr) +{ + struct bio_queue_data *bio_queue_data = ptr; + + blk_start_plug(&bio_queue_data->plug); +} + +static void finish_bio_queue(void *ptr) +{ + struct bio_queue_data *bio_queue_data = ptr; + + blk_finish_plug(&bio_queue_data->plug); +} + +static const struct vdo_work_queue_type bio_queue_type = { + .start = start_bio_queue, + .finish = finish_bio_queue, + .max_priority = BIO_Q_MAX_PRIORITY, + .default_priority = BIO_Q_DATA_PRIORITY, +}; + +/** + * count_all_bios() - Determine which bio counter to use. + * @vio: The vio associated with the bio. + * @bio: The bio to count. + */ +static void count_all_bios(struct vio *vio, struct bio *bio) +{ + struct atomic_statistics *stats = &vio->completion.vdo->stats; + + if (is_data_vio(vio)) { + vdo_count_bios(&stats->bios_out, bio); + return; + } + + vdo_count_bios(&stats->bios_meta, bio); + if (vio->type == VIO_TYPE_RECOVERY_JOURNAL) + vdo_count_bios(&stats->bios_journal, bio); + else if (vio->type == VIO_TYPE_BLOCK_MAP) + vdo_count_bios(&stats->bios_page_cache, bio); +} + +/** + * assert_in_bio_zone() - Assert that a vio is in the correct bio zone and not in interrupt + * context. + * @vio: The vio to check. + */ +static void assert_in_bio_zone(struct vio *vio) +{ + VDO_ASSERT_LOG_ONLY(!in_interrupt(), "not in interrupt context"); + assert_vio_in_bio_zone(vio); +} + +/** + * send_bio_to_device() - Update stats and tracing info, then submit the supplied bio to the OS for + * processing. + * @vio: The vio associated with the bio. + * @bio: The bio to submit to the OS. + */ +static void send_bio_to_device(struct vio *vio, struct bio *bio) +{ + struct vdo *vdo = vio->completion.vdo; + + assert_in_bio_zone(vio); + atomic64_inc(&vdo->stats.bios_submitted); + count_all_bios(vio, bio); + bio_set_dev(bio, vdo_get_backing_device(vdo)); + submit_bio_noacct(bio); +} + +/** + * vdo_submit_vio() - Submits a vio's bio to the underlying block device. May block if the device + * is busy. This callback should be used by vios which did not attempt to merge. + */ +void vdo_submit_vio(struct vdo_completion *completion) +{ + struct vio *vio = as_vio(completion); + + send_bio_to_device(vio, vio->bio); +} + +/** + * get_bio_list() - Extract the list of bios to submit from a vio. + * @vio: The vio submitting I/O. + * + * The list will always contain at least one entry (the bio for the vio on which it is called), but + * other bios may have been merged with it as well. + * + * Return: bio The head of the bio list to submit. + */ +static struct bio *get_bio_list(struct vio *vio) +{ + struct bio *bio; + struct io_submitter *submitter = vio->completion.vdo->io_submitter; + struct bio_queue_data *bio_queue_data = &(submitter->bio_queue_data[vio->bio_zone]); + + assert_in_bio_zone(vio); + + mutex_lock(&bio_queue_data->lock); + vdo_int_map_remove(bio_queue_data->map, + vio->bios_merged.head->bi_iter.bi_sector); + vdo_int_map_remove(bio_queue_data->map, + vio->bios_merged.tail->bi_iter.bi_sector); + bio = vio->bios_merged.head; + bio_list_init(&vio->bios_merged); + mutex_unlock(&bio_queue_data->lock); + + return bio; +} + +/** + * submit_data_vio() - Submit a data_vio's bio to the storage below along with + * any bios that have been merged with it. + * + * Context: This call may block and so should only be called from a bio thread. + */ +static void submit_data_vio(struct vdo_completion *completion) +{ + struct bio *bio, *next; + struct vio *vio = as_vio(completion); + + assert_in_bio_zone(vio); + for (bio = get_bio_list(vio); bio != NULL; bio = next) { + next = bio->bi_next; + bio->bi_next = NULL; + send_bio_to_device((struct vio *) bio->bi_private, bio); + } +} + +/** + * get_mergeable_locked() - Attempt to find an already queued bio that the current bio can be + * merged with. + * @map: The bio map to use for merging. + * @vio: The vio we want to merge. + * @back_merge: Set to true for a back merge, false for a front merge. + * + * There are two types of merging possible, forward and backward, which are distinguished by a flag + * that uses kernel elevator terminology. + * + * Return: the vio to merge to, NULL if no merging is possible. + */ +static struct vio *get_mergeable_locked(struct int_map *map, struct vio *vio, + bool back_merge) +{ + struct bio *bio = vio->bio; + sector_t merge_sector = bio->bi_iter.bi_sector; + struct vio *vio_merge; + + if (back_merge) + merge_sector -= VDO_SECTORS_PER_BLOCK; + else + merge_sector += VDO_SECTORS_PER_BLOCK; + + vio_merge = vdo_int_map_get(map, merge_sector); + + if (vio_merge == NULL) + return NULL; + + if (vio->completion.priority != vio_merge->completion.priority) + return NULL; + + if (bio_data_dir(bio) != bio_data_dir(vio_merge->bio)) + return NULL; + + if (bio_list_empty(&vio_merge->bios_merged)) + return NULL; + + if (back_merge) { + return (vio_merge->bios_merged.tail->bi_iter.bi_sector == merge_sector ? + vio_merge : NULL); + } + + return (vio_merge->bios_merged.head->bi_iter.bi_sector == merge_sector ? + vio_merge : NULL); +} + +static int map_merged_vio(struct int_map *bio_map, struct vio *vio) +{ + int result; + sector_t bio_sector; + + bio_sector = vio->bios_merged.head->bi_iter.bi_sector; + result = vdo_int_map_put(bio_map, bio_sector, vio, true, NULL); + if (result != VDO_SUCCESS) + return result; + + bio_sector = vio->bios_merged.tail->bi_iter.bi_sector; + return vdo_int_map_put(bio_map, bio_sector, vio, true, NULL); +} + +static int merge_to_prev_tail(struct int_map *bio_map, struct vio *vio, + struct vio *prev_vio) +{ + vdo_int_map_remove(bio_map, prev_vio->bios_merged.tail->bi_iter.bi_sector); + bio_list_merge(&prev_vio->bios_merged, &vio->bios_merged); + return map_merged_vio(bio_map, prev_vio); +} + +static int merge_to_next_head(struct int_map *bio_map, struct vio *vio, + struct vio *next_vio) +{ + /* + * Handle "next merge" and "gap fill" cases the same way so as to reorder bios in a way + * that's compatible with using funnel queues in work queues. This avoids removing an + * existing completion. + */ + vdo_int_map_remove(bio_map, next_vio->bios_merged.head->bi_iter.bi_sector); + bio_list_merge_head(&next_vio->bios_merged, &vio->bios_merged); + return map_merged_vio(bio_map, next_vio); +} + +/** + * try_bio_map_merge() - Attempt to merge a vio's bio with other pending I/Os. + * @vio: The vio to merge. + * + * Currently this is only used for data_vios, but is broken out for future use with metadata vios. + * + * Return: whether or not the vio was merged. + */ +static bool try_bio_map_merge(struct vio *vio) +{ + int result; + bool merged = true; + struct bio *bio = vio->bio; + struct vio *prev_vio, *next_vio; + struct vdo *vdo = vio->completion.vdo; + struct bio_queue_data *bio_queue_data = + &vdo->io_submitter->bio_queue_data[vio->bio_zone]; + + bio->bi_next = NULL; + bio_list_init(&vio->bios_merged); + bio_list_add(&vio->bios_merged, bio); + + mutex_lock(&bio_queue_data->lock); + prev_vio = get_mergeable_locked(bio_queue_data->map, vio, true); + next_vio = get_mergeable_locked(bio_queue_data->map, vio, false); + if (prev_vio == next_vio) + next_vio = NULL; + + if ((prev_vio == NULL) && (next_vio == NULL)) { + /* no merge. just add to bio_queue */ + merged = false; + result = vdo_int_map_put(bio_queue_data->map, + bio->bi_iter.bi_sector, + vio, true, NULL); + } else if (next_vio == NULL) { + /* Only prev. merge to prev's tail */ + result = merge_to_prev_tail(bio_queue_data->map, vio, prev_vio); + } else { + /* Only next. merge to next's head */ + result = merge_to_next_head(bio_queue_data->map, vio, next_vio); + } + mutex_unlock(&bio_queue_data->lock); + + /* We don't care about failure of int_map_put in this case. */ + VDO_ASSERT_LOG_ONLY(result == VDO_SUCCESS, "bio map insertion succeeds"); + return merged; +} + +/** + * vdo_submit_data_vio() - Submit I/O for a data_vio. + * @data_vio: the data_vio for which to issue I/O. + * + * If possible, this I/O will be merged other pending I/Os. Otherwise, the data_vio will be sent to + * the appropriate bio zone directly. + */ +void vdo_submit_data_vio(struct data_vio *data_vio) +{ + if (try_bio_map_merge(&data_vio->vio)) + return; + + launch_data_vio_bio_zone_callback(data_vio, submit_data_vio); +} + +/** + * __submit_metadata_vio() - Submit I/O for a metadata vio. + * @vio: the vio for which to issue I/O + * @physical: the physical block number to read or write + * @callback: the bio endio function which will be called after the I/O completes + * @error_handler: the handler for submission or I/O errors (may be NULL) + * @operation: the type of I/O to perform + * @data: the buffer to read or write (may be NULL) + * + * The vio is enqueued on a vdo bio queue so that bio submission (which may block) does not block + * other vdo threads. + * + * That the error handler will run on the correct thread is only true so long as the thread calling + * this function, and the thread set in the endio callback are the same, as well as the fact that + * no error can occur on the bio queue. Currently this is true for all callers, but additional care + * will be needed if this ever changes. + */ +void __submit_metadata_vio(struct vio *vio, physical_block_number_t physical, + bio_end_io_t callback, vdo_action_fn error_handler, + blk_opf_t operation, char *data) +{ + int result; + struct vdo_completion *completion = &vio->completion; + const struct admin_state_code *code = vdo_get_admin_state(completion->vdo); + + + VDO_ASSERT_LOG_ONLY(!code->quiescent, "I/O not allowed in state %s", code->name); + VDO_ASSERT_LOG_ONLY(vio->bio->bi_next == NULL, "metadata bio has no next bio"); + + vdo_reset_completion(completion); + completion->error_handler = error_handler; + result = vio_reset_bio(vio, data, callback, operation | REQ_META, physical); + if (result != VDO_SUCCESS) { + continue_vio(vio, result); + return; + } + + vdo_set_completion_callback(completion, vdo_submit_vio, + get_vio_bio_zone_thread_id(vio)); + vdo_launch_completion_with_priority(completion, get_metadata_priority(vio)); +} + +/** + * vdo_make_io_submitter() - Create an io_submitter structure. + * @thread_count: Number of bio-submission threads to set up. + * @rotation_interval: Interval to use when rotating between bio-submission threads when enqueuing + * completions. + * @max_requests_active: Number of bios for merge tracking. + * @vdo: The vdo which will use this submitter. + * @io_submitter: pointer to the new data structure. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_make_io_submitter(unsigned int thread_count, unsigned int rotation_interval, + unsigned int max_requests_active, struct vdo *vdo, + struct io_submitter **io_submitter_ptr) +{ + unsigned int i; + struct io_submitter *io_submitter; + int result; + + result = vdo_allocate_extended(struct io_submitter, thread_count, + struct bio_queue_data, "bio submission data", + &io_submitter); + if (result != VDO_SUCCESS) + return result; + + io_submitter->bio_queue_rotation_interval = rotation_interval; + + /* Setup for each bio-submission work queue */ + for (i = 0; i < thread_count; i++) { + struct bio_queue_data *bio_queue_data = &io_submitter->bio_queue_data[i]; + + mutex_init(&bio_queue_data->lock); + /* + * One I/O operation per request, but both first & last sector numbers. + * + * If requests are assigned to threads round-robin, they should be distributed + * quite evenly. But if they're assigned based on PBN, things can sometimes be very + * uneven. So for now, we'll assume that all requests *may* wind up on one thread, + * and thus all in the same map. + */ + result = vdo_int_map_create(max_requests_active * 2, + &bio_queue_data->map); + if (result != VDO_SUCCESS) { + /* + * Clean up the partially initialized bio-queue entirely and indicate that + * initialization failed. + */ + vdo_log_error("bio map initialization failed %d", result); + vdo_cleanup_io_submitter(io_submitter); + vdo_free_io_submitter(io_submitter); + return result; + } + + bio_queue_data->queue_number = i; + result = vdo_make_thread(vdo, vdo->thread_config.bio_threads[i], + &bio_queue_type, 1, (void **) &bio_queue_data); + if (result != VDO_SUCCESS) { + /* + * Clean up the partially initialized bio-queue entirely and indicate that + * initialization failed. + */ + vdo_int_map_free(vdo_forget(bio_queue_data->map)); + vdo_log_error("bio queue initialization failed %d", result); + vdo_cleanup_io_submitter(io_submitter); + vdo_free_io_submitter(io_submitter); + return result; + } + + bio_queue_data->queue = vdo->threads[vdo->thread_config.bio_threads[i]].queue; + io_submitter->num_bio_queues_used++; + } + + *io_submitter_ptr = io_submitter; + + return VDO_SUCCESS; +} + +/** + * vdo_cleanup_io_submitter() - Tear down the io_submitter fields as needed for a physical layer. + * @io_submitter: The I/O submitter data to tear down (may be NULL). + */ +void vdo_cleanup_io_submitter(struct io_submitter *io_submitter) +{ + int i; + + if (io_submitter == NULL) + return; + + for (i = io_submitter->num_bio_queues_used - 1; i >= 0; i--) + vdo_finish_work_queue(io_submitter->bio_queue_data[i].queue); +} + +/** + * vdo_free_io_submitter() - Free the io_submitter fields and structure as needed. + * @io_submitter: The I/O submitter data to destroy. + * + * This must be called after vdo_cleanup_io_submitter(). It is used to release resources late in + * the shutdown process to avoid or reduce the chance of race conditions. + */ +void vdo_free_io_submitter(struct io_submitter *io_submitter) +{ + int i; + + if (io_submitter == NULL) + return; + + for (i = io_submitter->num_bio_queues_used - 1; i >= 0; i--) { + io_submitter->num_bio_queues_used--; + /* vdo_destroy() will free the work queue, so just give up our reference to it. */ + vdo_forget(io_submitter->bio_queue_data[i].queue); + vdo_int_map_free(vdo_forget(io_submitter->bio_queue_data[i].map)); + } + vdo_free(io_submitter); +} diff --git a/drivers/md/dm-vdo/io-submitter.h b/drivers/md/dm-vdo/io-submitter.h new file mode 100644 index 000000000000..80748699496f --- /dev/null +++ b/drivers/md/dm-vdo/io-submitter.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_IO_SUBMITTER_H +#define VDO_IO_SUBMITTER_H + +#include + +#include "types.h" + +struct io_submitter; + +int vdo_make_io_submitter(unsigned int thread_count, unsigned int rotation_interval, + unsigned int max_requests_active, struct vdo *vdo, + struct io_submitter **io_submitter); + +void vdo_cleanup_io_submitter(struct io_submitter *io_submitter); + +void vdo_free_io_submitter(struct io_submitter *io_submitter); + +void vdo_submit_vio(struct vdo_completion *completion); + +void vdo_submit_data_vio(struct data_vio *data_vio); + +void __submit_metadata_vio(struct vio *vio, physical_block_number_t physical, + bio_end_io_t callback, vdo_action_fn error_handler, + blk_opf_t operation, char *data); + +static inline void vdo_submit_metadata_vio(struct vio *vio, physical_block_number_t physical, + bio_end_io_t callback, vdo_action_fn error_handler, + blk_opf_t operation) +{ + __submit_metadata_vio(vio, physical, callback, error_handler, + operation, vio->data); +} + +static inline void vdo_submit_flush_vio(struct vio *vio, bio_end_io_t callback, + vdo_action_fn error_handler) +{ + /* FIXME: Can we just use REQ_OP_FLUSH? */ + __submit_metadata_vio(vio, 0, callback, error_handler, + REQ_OP_WRITE | REQ_PREFLUSH, NULL); +} + +#endif /* VDO_IO_SUBMITTER_H */ diff --git a/drivers/md/dm-vdo/logger.c b/drivers/md/dm-vdo/logger.c new file mode 100644 index 000000000000..3f7dc2cb6b98 --- /dev/null +++ b/drivers/md/dm-vdo/logger.c @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "logger.h" + +#include +#include +#include +#include +#include +#include + +#include "errors.h" +#include "thread-device.h" +#include "thread-utils.h" + +int vdo_log_level = VDO_LOG_DEFAULT; + +int vdo_get_log_level(void) +{ + int log_level_latch = READ_ONCE(vdo_log_level); + + if (unlikely(log_level_latch > VDO_LOG_MAX)) { + log_level_latch = VDO_LOG_DEFAULT; + WRITE_ONCE(vdo_log_level, log_level_latch); + } + return log_level_latch; +} + +static const char *get_current_interrupt_type(void) +{ + if (in_nmi()) + return "NMI"; + + if (in_irq()) + return "HI"; + + if (in_softirq()) + return "SI"; + + return "INTR"; +} + +/** + * emit_log_message_to_kernel() - Emit a log message to the kernel at the specified priority. + * + * @priority: The priority at which to log the message + * @fmt: The format string of the message + */ +static void emit_log_message_to_kernel(int priority, const char *fmt, ...) +{ + va_list args; + struct va_format vaf; + + if (priority > vdo_get_log_level()) + return; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + + switch (priority) { + case VDO_LOG_EMERG: + case VDO_LOG_ALERT: + case VDO_LOG_CRIT: + pr_crit("%pV", &vaf); + break; + case VDO_LOG_ERR: + pr_err("%pV", &vaf); + break; + case VDO_LOG_WARNING: + pr_warn("%pV", &vaf); + break; + case VDO_LOG_NOTICE: + case VDO_LOG_INFO: + pr_info("%pV", &vaf); + break; + case VDO_LOG_DEBUG: + pr_debug("%pV", &vaf); + break; + default: + printk(KERN_DEFAULT "%pV", &vaf); + break; + } + + va_end(args); +} + +/** + * emit_log_message() - Emit a log message to the kernel log in a format suited to the current + * thread context. + * + * Context info formats: + * + * interrupt: uds[NMI]: blah + * kvdo thread: kvdo12:foobarQ: blah + * thread w/device id: kvdo12:myprog: blah + * other thread: uds: myprog: blah + * + * Fields: module name, interrupt level, process name, device ID. + * + * @priority: the priority at which to log the message + * @module: The name of the module doing the logging + * @prefix: The prefix of the log message + * @vaf1: The first message format descriptor + * @vaf2: The second message format descriptor + */ +static void emit_log_message(int priority, const char *module, const char *prefix, + const struct va_format *vaf1, const struct va_format *vaf2) +{ + int device_instance; + + /* + * In interrupt context, identify the interrupt type and module. Ignore the process/thread + * since it could be anything. + */ + if (in_interrupt()) { + const char *type = get_current_interrupt_type(); + + emit_log_message_to_kernel(priority, "%s[%s]: %s%pV%pV\n", module, type, + prefix, vaf1, vaf2); + return; + } + + /* Not at interrupt level; we have a process we can look at, and might have a device ID. */ + device_instance = vdo_get_thread_device_id(); + if (device_instance >= 0) { + emit_log_message_to_kernel(priority, "%s%u:%s: %s%pV%pV\n", module, + device_instance, current->comm, prefix, vaf1, + vaf2); + return; + } + + /* + * If it's a kernel thread and the module name is a prefix of its name, assume it is ours + * and only identify the thread. + */ + if (((current->flags & PF_KTHREAD) != 0) && + (strncmp(module, current->comm, strlen(module)) == 0)) { + emit_log_message_to_kernel(priority, "%s: %s%pV%pV\n", current->comm, + prefix, vaf1, vaf2); + return; + } + + /* Identify the module and the process. */ + emit_log_message_to_kernel(priority, "%s: %s: %s%pV%pV\n", module, current->comm, + prefix, vaf1, vaf2); +} + +/* + * vdo_log_embedded_message() - Log a message embedded within another message. + * @priority: the priority at which to log the message + * @module: the name of the module doing the logging + * @prefix: optional string prefix to message, may be NULL + * @fmt1: format of message first part (required) + * @args1: arguments for message first part (required) + * @fmt2: format of message second part + */ +void vdo_log_embedded_message(int priority, const char *module, const char *prefix, + const char *fmt1, va_list args1, const char *fmt2, ...) +{ + va_list args1_copy; + va_list args2; + struct va_format vaf1, vaf2; + + va_start(args2, fmt2); + + if (module == NULL) + module = VDO_LOGGING_MODULE_NAME; + + if (prefix == NULL) + prefix = ""; + + /* + * It is implementation dependent whether va_list is defined as an array type that decays + * to a pointer when passed as an argument. Copy args1 and args2 with va_copy so that vaf1 + * and vaf2 get proper va_list pointers irrespective of how va_list is defined. + */ + va_copy(args1_copy, args1); + vaf1.fmt = fmt1; + vaf1.va = &args1_copy; + + vaf2.fmt = fmt2; + vaf2.va = &args2; + + emit_log_message(priority, module, prefix, &vaf1, &vaf2); + + va_end(args1_copy); + va_end(args2); +} + +int vdo_vlog_strerror(int priority, int errnum, const char *module, const char *format, + va_list args) +{ + char errbuf[VDO_MAX_ERROR_MESSAGE_SIZE]; + const char *message = uds_string_error(errnum, errbuf, sizeof(errbuf)); + + vdo_log_embedded_message(priority, module, NULL, format, args, ": %s (%d)", + message, errnum); + return errnum; +} + +int __vdo_log_strerror(int priority, int errnum, const char *module, const char *format, ...) +{ + va_list args; + + va_start(args, format); + vdo_vlog_strerror(priority, errnum, module, format, args); + va_end(args); + return errnum; +} + +void vdo_log_backtrace(int priority) +{ + if (priority > vdo_get_log_level()) + return; + + dump_stack(); +} + +void __vdo_log_message(int priority, const char *module, const char *format, ...) +{ + va_list args; + + va_start(args, format); + vdo_log_embedded_message(priority, module, NULL, format, args, "%s", ""); + va_end(args); +} + +/* + * Sleep or delay a few milliseconds in an attempt to allow the log buffers to be flushed lest they + * be overrun. + */ +void vdo_pause_for_logger(void) +{ + fsleep(4000); +} diff --git a/drivers/md/dm-vdo/logger.h b/drivers/md/dm-vdo/logger.h new file mode 100644 index 000000000000..ae6ad691c027 --- /dev/null +++ b/drivers/md/dm-vdo/logger.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_LOGGER_H +#define VDO_LOGGER_H + +#include +#include +#include +#include + +/* Custom logging utilities for UDS */ + +enum { + VDO_LOG_EMERG = LOGLEVEL_EMERG, + VDO_LOG_ALERT = LOGLEVEL_ALERT, + VDO_LOG_CRIT = LOGLEVEL_CRIT, + VDO_LOG_ERR = LOGLEVEL_ERR, + VDO_LOG_WARNING = LOGLEVEL_WARNING, + VDO_LOG_NOTICE = LOGLEVEL_NOTICE, + VDO_LOG_INFO = LOGLEVEL_INFO, + VDO_LOG_DEBUG = LOGLEVEL_DEBUG, + + VDO_LOG_MAX = VDO_LOG_DEBUG, + VDO_LOG_DEFAULT = VDO_LOG_INFO, +}; + +extern int vdo_log_level; + +#define DM_MSG_PREFIX "vdo" +#define VDO_LOGGING_MODULE_NAME DM_NAME ": " DM_MSG_PREFIX + +/* Apply a rate limiter to a log method call. */ +#define vdo_log_ratelimit(log_fn, ...) \ + do { \ + static DEFINE_RATELIMIT_STATE(_rs, \ + DEFAULT_RATELIMIT_INTERVAL, \ + DEFAULT_RATELIMIT_BURST); \ + if (__ratelimit(&_rs)) { \ + log_fn(__VA_ARGS__); \ + } \ + } while (0) + +int vdo_get_log_level(void); + +void vdo_log_embedded_message(int priority, const char *module, const char *prefix, + const char *fmt1, va_list args1, const char *fmt2, ...) + __printf(4, 0) __printf(6, 7); + +void vdo_log_backtrace(int priority); + +/* All log functions will preserve the caller's value of errno. */ + +#define vdo_log_strerror(priority, errnum, ...) \ + __vdo_log_strerror(priority, errnum, VDO_LOGGING_MODULE_NAME, __VA_ARGS__) + +int __vdo_log_strerror(int priority, int errnum, const char *module, + const char *format, ...) + __printf(4, 5); + +int vdo_vlog_strerror(int priority, int errnum, const char *module, const char *format, + va_list args) + __printf(4, 0); + +/* Log an error prefixed with the string associated with the errnum. */ +#define vdo_log_error_strerror(errnum, ...) \ + vdo_log_strerror(VDO_LOG_ERR, errnum, __VA_ARGS__) + +#define vdo_log_debug_strerror(errnum, ...) \ + vdo_log_strerror(VDO_LOG_DEBUG, errnum, __VA_ARGS__) + +#define vdo_log_info_strerror(errnum, ...) \ + vdo_log_strerror(VDO_LOG_INFO, errnum, __VA_ARGS__) + +#define vdo_log_warning_strerror(errnum, ...) \ + vdo_log_strerror(VDO_LOG_WARNING, errnum, __VA_ARGS__) + +#define vdo_log_fatal_strerror(errnum, ...) \ + vdo_log_strerror(VDO_LOG_CRIT, errnum, __VA_ARGS__) + +#define vdo_log_message(priority, ...) \ + __vdo_log_message(priority, VDO_LOGGING_MODULE_NAME, __VA_ARGS__) + +void __vdo_log_message(int priority, const char *module, const char *format, ...) + __printf(3, 4); + +#define vdo_log_debug(...) vdo_log_message(VDO_LOG_DEBUG, __VA_ARGS__) + +#define vdo_log_info(...) vdo_log_message(VDO_LOG_INFO, __VA_ARGS__) + +#define vdo_log_warning(...) vdo_log_message(VDO_LOG_WARNING, __VA_ARGS__) + +#define vdo_log_error(...) vdo_log_message(VDO_LOG_ERR, __VA_ARGS__) + +#define vdo_log_fatal(...) vdo_log_message(VDO_LOG_CRIT, __VA_ARGS__) + +void vdo_pause_for_logger(void); +#endif /* VDO_LOGGER_H */ diff --git a/drivers/md/dm-vdo/logical-zone.c b/drivers/md/dm-vdo/logical-zone.c new file mode 100644 index 000000000000..026f031ffc9e --- /dev/null +++ b/drivers/md/dm-vdo/logical-zone.c @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "logical-zone.h" + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" +#include "string-utils.h" + +#include "action-manager.h" +#include "admin-state.h" +#include "block-map.h" +#include "completion.h" +#include "constants.h" +#include "data-vio.h" +#include "flush.h" +#include "int-map.h" +#include "physical-zone.h" +#include "vdo.h" + +#define ALLOCATIONS_PER_ZONE 128 + +/** + * as_logical_zone() - Convert a generic vdo_completion to a logical_zone. + * @completion: The completion to convert. + * + * Return: The completion as a logical_zone. + */ +static struct logical_zone *as_logical_zone(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_GENERATION_FLUSHED_COMPLETION); + return container_of(completion, struct logical_zone, completion); +} + +/* get_thread_id_for_zone() - Implements vdo_zone_thread_getter_fn. */ +static thread_id_t get_thread_id_for_zone(void *context, zone_count_t zone_number) +{ + struct logical_zones *zones = context; + + return zones->zones[zone_number].thread_id; +} + +/** + * initialize_zone() - Initialize a logical zone. + * @zones: The logical_zones to which this zone belongs. + * @zone_number: The logical_zone's index. + */ +static int initialize_zone(struct logical_zones *zones, zone_count_t zone_number) +{ + int result; + struct vdo *vdo = zones->vdo; + struct logical_zone *zone = &zones->zones[zone_number]; + zone_count_t allocation_zone_number; + + result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, &zone->lbn_operations); + if (result != VDO_SUCCESS) + return result; + + if (zone_number < vdo->thread_config.logical_zone_count - 1) + zone->next = &zones->zones[zone_number + 1]; + + vdo_initialize_completion(&zone->completion, vdo, + VDO_GENERATION_FLUSHED_COMPLETION); + zone->zones = zones; + zone->zone_number = zone_number; + zone->thread_id = vdo->thread_config.logical_threads[zone_number]; + zone->block_map_zone = &vdo->block_map->zones[zone_number]; + INIT_LIST_HEAD(&zone->write_vios); + vdo_set_admin_state_code(&zone->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + + allocation_zone_number = zone->thread_id % vdo->thread_config.physical_zone_count; + zone->allocation_zone = &vdo->physical_zones->zones[allocation_zone_number]; + + return vdo_make_default_thread(vdo, zone->thread_id); +} + +/** + * vdo_make_logical_zones() - Create a set of logical zones. + * @vdo: The vdo to which the zones will belong. + * @zones_ptr: A pointer to hold the new zones. + * + * Return: VDO_SUCCESS or an error code. + */ +int vdo_make_logical_zones(struct vdo *vdo, struct logical_zones **zones_ptr) +{ + struct logical_zones *zones; + int result; + zone_count_t zone; + zone_count_t zone_count = vdo->thread_config.logical_zone_count; + + if (zone_count == 0) + return VDO_SUCCESS; + + result = vdo_allocate_extended(struct logical_zones, zone_count, + struct logical_zone, __func__, &zones); + if (result != VDO_SUCCESS) + return result; + + zones->vdo = vdo; + zones->zone_count = zone_count; + for (zone = 0; zone < zone_count; zone++) { + result = initialize_zone(zones, zone); + if (result != VDO_SUCCESS) { + vdo_free_logical_zones(zones); + return result; + } + } + + result = vdo_make_action_manager(zones->zone_count, get_thread_id_for_zone, + vdo->thread_config.admin_thread, zones, NULL, + vdo, &zones->manager); + if (result != VDO_SUCCESS) { + vdo_free_logical_zones(zones); + return result; + } + + *zones_ptr = zones; + return VDO_SUCCESS; +} + +/** + * vdo_free_logical_zones() - Free a set of logical zones. + * @zones: The set of zones to free. + */ +void vdo_free_logical_zones(struct logical_zones *zones) +{ + zone_count_t index; + + if (zones == NULL) + return; + + vdo_free(vdo_forget(zones->manager)); + + for (index = 0; index < zones->zone_count; index++) + vdo_int_map_free(vdo_forget(zones->zones[index].lbn_operations)); + + vdo_free(zones); +} + +static inline void assert_on_zone_thread(struct logical_zone *zone, const char *what) +{ + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == zone->thread_id), + "%s() called on correct thread", what); +} + +/** + * check_for_drain_complete() - Check whether this zone has drained. + * @zone: The zone to check. + */ +static void check_for_drain_complete(struct logical_zone *zone) +{ + if (!vdo_is_state_draining(&zone->state) || zone->notifying || + !list_empty(&zone->write_vios)) + return; + + vdo_finish_draining(&zone->state); +} + +/** + * initiate_drain() - Initiate a drain. + * + * Implements vdo_admin_initiator_fn. + */ +static void initiate_drain(struct admin_state *state) +{ + check_for_drain_complete(container_of(state, struct logical_zone, state)); +} + +/** + * drain_logical_zone() - Drain a logical zone. + * + * Implements vdo_zone_action_fn. + */ +static void drain_logical_zone(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct logical_zones *zones = context; + + vdo_start_draining(&zones->zones[zone_number].state, + vdo_get_current_manager_operation(zones->manager), parent, + initiate_drain); +} + +void vdo_drain_logical_zones(struct logical_zones *zones, + const struct admin_state_code *operation, + struct vdo_completion *parent) +{ + vdo_schedule_operation(zones->manager, operation, NULL, drain_logical_zone, NULL, + parent); +} + +/** + * resume_logical_zone() - Resume a logical zone. + * + * Implements vdo_zone_action_fn. + */ +static void resume_logical_zone(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct logical_zone *zone = &(((struct logical_zones *) context)->zones[zone_number]); + + vdo_fail_completion(parent, vdo_resume_if_quiescent(&zone->state)); +} + +/** + * vdo_resume_logical_zones() - Resume a set of logical zones. + * @zones: The logical zones to resume. + * @parent: The object to notify when the zones have resumed. + */ +void vdo_resume_logical_zones(struct logical_zones *zones, struct vdo_completion *parent) +{ + vdo_schedule_operation(zones->manager, VDO_ADMIN_STATE_RESUMING, NULL, + resume_logical_zone, NULL, parent); +} + +/** + * update_oldest_active_generation() - Update the oldest active generation. + * @zone: The zone. + * + * Return: true if the oldest active generation has changed. + */ +static bool update_oldest_active_generation(struct logical_zone *zone) +{ + struct data_vio *data_vio = + list_first_entry_or_null(&zone->write_vios, struct data_vio, + write_entry); + sequence_number_t oldest = + (data_vio == NULL) ? zone->flush_generation : data_vio->flush_generation; + + if (oldest == zone->oldest_active_generation) + return false; + + WRITE_ONCE(zone->oldest_active_generation, oldest); + return true; +} + +/** + * vdo_increment_logical_zone_flush_generation() - Increment the flush generation in a logical + * zone. + * @zone: The logical zone. + * @expected_generation: The expected value of the flush generation before the increment. + */ +void vdo_increment_logical_zone_flush_generation(struct logical_zone *zone, + sequence_number_t expected_generation) +{ + assert_on_zone_thread(zone, __func__); + VDO_ASSERT_LOG_ONLY((zone->flush_generation == expected_generation), + "logical zone %u flush generation %llu should be %llu before increment", + zone->zone_number, (unsigned long long) zone->flush_generation, + (unsigned long long) expected_generation); + + zone->flush_generation++; + zone->ios_in_flush_generation = 0; + update_oldest_active_generation(zone); +} + +/** + * vdo_acquire_flush_generation_lock() - Acquire the shared lock on a flush generation by a write + * data_vio. + * @data_vio: The data_vio. + */ +void vdo_acquire_flush_generation_lock(struct data_vio *data_vio) +{ + struct logical_zone *zone = data_vio->logical.zone; + + assert_on_zone_thread(zone, __func__); + VDO_ASSERT_LOG_ONLY(vdo_is_state_normal(&zone->state), "vdo state is normal"); + + data_vio->flush_generation = zone->flush_generation; + list_add_tail(&data_vio->write_entry, &zone->write_vios); + zone->ios_in_flush_generation++; +} + +static void attempt_generation_complete_notification(struct vdo_completion *completion); + +/** + * notify_flusher() - Notify the flush that at least one generation no longer has active VIOs. + * @completion: The zone completion. + * + * This callback is registered in attempt_generation_complete_notification(). + */ +static void notify_flusher(struct vdo_completion *completion) +{ + struct logical_zone *zone = as_logical_zone(completion); + + vdo_complete_flushes(zone->zones->vdo->flusher); + vdo_launch_completion_callback(completion, + attempt_generation_complete_notification, + zone->thread_id); +} + +/** + * attempt_generation_complete_notification() - Notify the flusher if some generation no + * longer has active VIOs. + * @completion: The zone completion. + */ +static void attempt_generation_complete_notification(struct vdo_completion *completion) +{ + struct logical_zone *zone = as_logical_zone(completion); + + assert_on_zone_thread(zone, __func__); + if (zone->oldest_active_generation <= zone->notification_generation) { + zone->notifying = false; + check_for_drain_complete(zone); + return; + } + + zone->notifying = true; + zone->notification_generation = zone->oldest_active_generation; + vdo_launch_completion_callback(&zone->completion, notify_flusher, + vdo_get_flusher_thread_id(zone->zones->vdo->flusher)); +} + +/** + * vdo_release_flush_generation_lock() - Release the shared lock on a flush generation held by a + * write data_vio. + * @data_vio: The data_vio whose lock is to be released. + * + * If there are pending flushes, and this data_vio completes the oldest generation active in this + * zone, an attempt will be made to finish any flushes which may now be complete. + */ +void vdo_release_flush_generation_lock(struct data_vio *data_vio) +{ + struct logical_zone *zone = data_vio->logical.zone; + + assert_on_zone_thread(zone, __func__); + + if (!data_vio_has_flush_generation_lock(data_vio)) + return; + + list_del_init(&data_vio->write_entry); + VDO_ASSERT_LOG_ONLY((zone->oldest_active_generation <= data_vio->flush_generation), + "data_vio releasing lock on generation %llu is not older than oldest active generation %llu", + (unsigned long long) data_vio->flush_generation, + (unsigned long long) zone->oldest_active_generation); + + if (!update_oldest_active_generation(zone) || zone->notifying) + return; + + attempt_generation_complete_notification(&zone->completion); +} + +struct physical_zone *vdo_get_next_allocation_zone(struct logical_zone *zone) +{ + if (zone->allocation_count == ALLOCATIONS_PER_ZONE) { + zone->allocation_count = 0; + zone->allocation_zone = zone->allocation_zone->next; + } + + zone->allocation_count++; + return zone->allocation_zone; +} + +/** + * vdo_dump_logical_zone() - Dump information about a logical zone to the log for debugging. + * @zone: The zone to dump + * + * Context: the information is dumped in a thread-unsafe fashion. + * + */ +void vdo_dump_logical_zone(const struct logical_zone *zone) +{ + vdo_log_info("logical_zone %u", zone->zone_number); + vdo_log_info(" flush_generation=%llu oldest_active_generation=%llu notification_generation=%llu notifying=%s ios_in_flush_generation=%llu", + (unsigned long long) READ_ONCE(zone->flush_generation), + (unsigned long long) READ_ONCE(zone->oldest_active_generation), + (unsigned long long) READ_ONCE(zone->notification_generation), + vdo_bool_to_string(READ_ONCE(zone->notifying)), + (unsigned long long) READ_ONCE(zone->ios_in_flush_generation)); +} diff --git a/drivers/md/dm-vdo/logical-zone.h b/drivers/md/dm-vdo/logical-zone.h new file mode 100644 index 000000000000..1b666c84a193 --- /dev/null +++ b/drivers/md/dm-vdo/logical-zone.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_LOGICAL_ZONE_H +#define VDO_LOGICAL_ZONE_H + +#include + +#include "admin-state.h" +#include "int-map.h" +#include "types.h" + +struct physical_zone; + +struct logical_zone { + /* The completion for flush notifications */ + struct vdo_completion completion; + /* The owner of this zone */ + struct logical_zones *zones; + /* Which logical zone this is */ + zone_count_t zone_number; + /* The thread id for this zone */ + thread_id_t thread_id; + /* In progress operations keyed by LBN */ + struct int_map *lbn_operations; + /* The logical to physical map */ + struct block_map_zone *block_map_zone; + /* The current flush generation */ + sequence_number_t flush_generation; + /* + * The oldest active generation in this zone. This is mutated only on the logical zone + * thread but is queried from the flusher thread. + */ + sequence_number_t oldest_active_generation; + /* The number of IOs in the current flush generation */ + block_count_t ios_in_flush_generation; + /* The youngest generation of the current notification */ + sequence_number_t notification_generation; + /* Whether a notification is in progress */ + bool notifying; + /* The queue of active data write VIOs */ + struct list_head write_vios; + /* The administrative state of the zone */ + struct admin_state state; + /* The physical zone from which to allocate */ + struct physical_zone *allocation_zone; + /* The number of allocations done from the current allocation_zone */ + block_count_t allocation_count; + /* The next zone */ + struct logical_zone *next; +}; + +struct logical_zones { + /* The vdo whose zones these are */ + struct vdo *vdo; + /* The manager for administrative actions */ + struct action_manager *manager; + /* The number of zones */ + zone_count_t zone_count; + /* The logical zones themselves */ + struct logical_zone zones[]; +}; + +int __must_check vdo_make_logical_zones(struct vdo *vdo, + struct logical_zones **zones_ptr); + +void vdo_free_logical_zones(struct logical_zones *zones); + +void vdo_drain_logical_zones(struct logical_zones *zones, + const struct admin_state_code *operation, + struct vdo_completion *completion); + +void vdo_resume_logical_zones(struct logical_zones *zones, + struct vdo_completion *parent); + +void vdo_increment_logical_zone_flush_generation(struct logical_zone *zone, + sequence_number_t expected_generation); + +void vdo_acquire_flush_generation_lock(struct data_vio *data_vio); + +void vdo_release_flush_generation_lock(struct data_vio *data_vio); + +struct physical_zone * __must_check vdo_get_next_allocation_zone(struct logical_zone *zone); + +void vdo_dump_logical_zone(const struct logical_zone *zone); + +#endif /* VDO_LOGICAL_ZONE_H */ diff --git a/drivers/md/dm-vdo/memory-alloc.c b/drivers/md/dm-vdo/memory-alloc.c new file mode 100644 index 000000000000..185f259c7245 --- /dev/null +++ b/drivers/md/dm-vdo/memory-alloc.c @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include +#include +#include +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +/* + * UDS and VDO keep track of which threads are allowed to allocate memory freely, and which threads + * must be careful to not do a memory allocation that does an I/O request. The 'allocating_threads' + * thread_registry and its associated methods implement this tracking. + */ +static struct thread_registry allocating_threads; + +static inline bool allocations_allowed(void) +{ + return vdo_lookup_thread(&allocating_threads) != NULL; +} + +/* + * Register the current thread as an allocating thread. + * + * An optional flag location can be supplied indicating whether, at any given point in time, the + * threads associated with that flag should be allocating storage. If the flag is false, a message + * will be logged. + * + * If no flag is supplied, the thread is always allowed to allocate storage without complaint. + * + * @new_thread: registered_thread structure to use for the current thread + * @flag_ptr: Location of the allocation-allowed flag + */ +void vdo_register_allocating_thread(struct registered_thread *new_thread, + const bool *flag_ptr) +{ + if (flag_ptr == NULL) { + static const bool allocation_always_allowed = true; + + flag_ptr = &allocation_always_allowed; + } + + vdo_register_thread(&allocating_threads, new_thread, flag_ptr); +} + +/* Unregister the current thread as an allocating thread. */ +void vdo_unregister_allocating_thread(void) +{ + vdo_unregister_thread(&allocating_threads); +} + +/* + * We track how much memory has been allocated and freed. When we unload the module, we log an + * error if we have not freed all the memory that we allocated. Nearly all memory allocation and + * freeing is done using this module. + * + * We do not use kernel functions like the kvasprintf() method, which allocate memory indirectly + * using kmalloc. + * + * These data structures and methods are used to track the amount of memory used. + */ + +/* + * We allocate very few large objects, and allocation/deallocation isn't done in a + * performance-critical stage for us, so a linked list should be fine. + */ +struct vmalloc_block_info { + void *ptr; + size_t size; + struct vmalloc_block_info *next; +}; + +static struct { + spinlock_t lock; + size_t kmalloc_blocks; + size_t kmalloc_bytes; + size_t vmalloc_blocks; + size_t vmalloc_bytes; + size_t peak_bytes; + struct vmalloc_block_info *vmalloc_list; +} memory_stats __cacheline_aligned; + +static void update_peak_usage(void) +{ + size_t total_bytes = memory_stats.kmalloc_bytes + memory_stats.vmalloc_bytes; + + if (total_bytes > memory_stats.peak_bytes) + memory_stats.peak_bytes = total_bytes; +} + +static void add_kmalloc_block(size_t size) +{ + unsigned long flags; + + spin_lock_irqsave(&memory_stats.lock, flags); + memory_stats.kmalloc_blocks++; + memory_stats.kmalloc_bytes += size; + update_peak_usage(); + spin_unlock_irqrestore(&memory_stats.lock, flags); +} + +static void remove_kmalloc_block(size_t size) +{ + unsigned long flags; + + spin_lock_irqsave(&memory_stats.lock, flags); + memory_stats.kmalloc_blocks--; + memory_stats.kmalloc_bytes -= size; + spin_unlock_irqrestore(&memory_stats.lock, flags); +} + +static void add_vmalloc_block(struct vmalloc_block_info *block) +{ + unsigned long flags; + + spin_lock_irqsave(&memory_stats.lock, flags); + block->next = memory_stats.vmalloc_list; + memory_stats.vmalloc_list = block; + memory_stats.vmalloc_blocks++; + memory_stats.vmalloc_bytes += block->size; + update_peak_usage(); + spin_unlock_irqrestore(&memory_stats.lock, flags); +} + +static void remove_vmalloc_block(void *ptr) +{ + struct vmalloc_block_info *block; + struct vmalloc_block_info **block_ptr; + unsigned long flags; + + spin_lock_irqsave(&memory_stats.lock, flags); + for (block_ptr = &memory_stats.vmalloc_list; + (block = *block_ptr) != NULL; + block_ptr = &block->next) { + if (block->ptr == ptr) { + *block_ptr = block->next; + memory_stats.vmalloc_blocks--; + memory_stats.vmalloc_bytes -= block->size; + break; + } + } + + spin_unlock_irqrestore(&memory_stats.lock, flags); + if (block != NULL) + vdo_free(block); + else + vdo_log_info("attempting to remove ptr %px not found in vmalloc list", ptr); +} + +/* + * Determine whether allocating a memory block should use kmalloc or __vmalloc. + * + * vmalloc can allocate any integral number of pages. + * + * kmalloc can allocate any number of bytes up to a configured limit, which defaults to 8 megabytes + * on some systems. kmalloc is especially good when memory is being both allocated and freed, and + * it does this efficiently in a multi CPU environment. + * + * kmalloc usually rounds the size of the block up to the next power of two, so when the requested + * block is bigger than PAGE_SIZE / 2 bytes, kmalloc will never give you less space than the + * corresponding vmalloc allocation. Sometimes vmalloc will use less overhead than kmalloc. + * + * The advantages of kmalloc do not help out UDS or VDO, because we allocate all our memory up + * front and do not free and reallocate it. Sometimes we have problems using kmalloc, because the + * Linux memory page map can become so fragmented that kmalloc will not give us a 32KB chunk. We + * have used vmalloc as a backup to kmalloc in the past, and a follow-up vmalloc of 32KB will work. + * But there is no strong case to be made for using kmalloc over vmalloc for these size chunks. + * + * The kmalloc/vmalloc boundary is set at 4KB, and kmalloc gets the 4KB requests. There is no + * strong reason for favoring either kmalloc or vmalloc for 4KB requests, except that tracking + * vmalloc statistics uses a linked list implementation. Using a simple test, this choice of + * boundary results in 132 vmalloc calls. Using vmalloc for requests of exactly 4KB results in an + * additional 6374 vmalloc calls, which is much less efficient for tracking. + * + * @size: How many bytes to allocate + */ +static inline bool use_kmalloc(size_t size) +{ + return size <= PAGE_SIZE; +} + +/* + * Allocate storage based on memory size and alignment, logging an error if the allocation fails. + * The memory will be zeroed. + * + * @size: The size of an object + * @align: The required alignment + * @what: What is being allocated (for error logging) + * @ptr: A pointer to hold the allocated memory + * + * Return: VDO_SUCCESS or an error code + */ +int vdo_allocate_memory(size_t size, size_t align, const char *what, void *ptr) +{ + /* + * The __GFP_RETRY_MAYFAIL flag means the VM implementation will retry memory reclaim + * procedures that have previously failed if there is some indication that progress has + * been made elsewhere. It can wait for other tasks to attempt high level approaches to + * freeing memory such as compaction (which removes fragmentation) and page-out. There is + * still a definite limit to the number of retries, but it is a larger limit than with + * __GFP_NORETRY. Allocations with this flag may fail, but only when there is genuinely + * little unused memory. While these allocations do not directly trigger the OOM killer, + * their failure indicates that the system is likely to need to use the OOM killer soon. + * The caller must handle failure, but can reasonably do so by failing a higher-level + * request, or completing it only in a much less efficient manner. + */ + const gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_RETRY_MAYFAIL; + unsigned int noio_flags; + bool allocations_restricted = !allocations_allowed(); + unsigned long start_time; + void *p = NULL; + + if (unlikely(ptr == NULL)) + return -EINVAL; + + if (size == 0) { + *((void **) ptr) = NULL; + return VDO_SUCCESS; + } + + if (allocations_restricted) + noio_flags = memalloc_noio_save(); + + start_time = jiffies; + if (use_kmalloc(size) && (align < PAGE_SIZE)) { + p = kmalloc(size, gfp_flags | __GFP_NOWARN); + if (p == NULL) { + /* + * It is possible for kmalloc to fail to allocate memory because there is + * no page available. A short sleep may allow the page reclaimer to + * free a page. + */ + fsleep(1000); + p = kmalloc(size, gfp_flags); + } + + if (p != NULL) + add_kmalloc_block(ksize(p)); + } else { + struct vmalloc_block_info *block; + + if (vdo_allocate(1, struct vmalloc_block_info, __func__, &block) == VDO_SUCCESS) { + /* + * It is possible for __vmalloc to fail to allocate memory because there + * are no pages available. A short sleep may allow the page reclaimer + * to free enough pages for a small allocation. + * + * For larger allocations, the page_alloc code is racing against the page + * reclaimer. If the page reclaimer can stay ahead of page_alloc, the + * __vmalloc will succeed. But if page_alloc overtakes the page reclaimer, + * the allocation fails. It is possible that more retries will succeed. + */ + for (;;) { + p = __vmalloc(size, gfp_flags | __GFP_NOWARN); + if (p != NULL) + break; + + if (jiffies_to_msecs(jiffies - start_time) > 1000) { + /* Try one more time, logging a failure for this call. */ + p = __vmalloc(size, gfp_flags); + break; + } + + fsleep(1000); + } + + if (p == NULL) { + vdo_free(block); + } else { + block->ptr = p; + block->size = PAGE_ALIGN(size); + add_vmalloc_block(block); + } + } + } + + if (allocations_restricted) + memalloc_noio_restore(noio_flags); + + if (unlikely(p == NULL)) { + vdo_log_error("Could not allocate %zu bytes for %s in %u msecs", + size, what, jiffies_to_msecs(jiffies - start_time)); + return -ENOMEM; + } + + *((void **) ptr) = p; + return VDO_SUCCESS; +} + +/* + * Allocate storage based on memory size, failing immediately if the required memory is not + * available. The memory will be zeroed. + * + * @size: The size of an object. + * @what: What is being allocated (for error logging) + * + * Return: pointer to the allocated memory, or NULL if the required space is not available. + */ +void *vdo_allocate_memory_nowait(size_t size, const char *what __maybe_unused) +{ + void *p = kmalloc(size, GFP_NOWAIT | __GFP_ZERO); + + if (p != NULL) + add_kmalloc_block(ksize(p)); + + return p; +} + +void vdo_free(void *ptr) +{ + if (ptr != NULL) { + if (is_vmalloc_addr(ptr)) { + remove_vmalloc_block(ptr); + vfree(ptr); + } else { + remove_kmalloc_block(ksize(ptr)); + kfree(ptr); + } + } +} + +/* + * Reallocate dynamically allocated memory. There are no alignment guarantees for the reallocated + * memory. If the new memory is larger than the old memory, the new space will be zeroed. + * + * @ptr: The memory to reallocate. + * @old_size: The old size of the memory + * @size: The new size to allocate + * @what: What is being allocated (for error logging) + * @new_ptr: A pointer to hold the reallocated pointer + * + * Return: VDO_SUCCESS or an error code + */ +int vdo_reallocate_memory(void *ptr, size_t old_size, size_t size, const char *what, + void *new_ptr) +{ + int result; + + if (size == 0) { + vdo_free(ptr); + *(void **) new_ptr = NULL; + return VDO_SUCCESS; + } + + result = vdo_allocate(size, char, what, new_ptr); + if (result != VDO_SUCCESS) + return result; + + if (ptr != NULL) { + if (old_size < size) + size = old_size; + + memcpy(*((void **) new_ptr), ptr, size); + vdo_free(ptr); + } + + return VDO_SUCCESS; +} + +int vdo_duplicate_string(const char *string, const char *what, char **new_string) +{ + int result; + u8 *dup; + + result = vdo_allocate(strlen(string) + 1, u8, what, &dup); + if (result != VDO_SUCCESS) + return result; + + memcpy(dup, string, strlen(string) + 1); + *new_string = dup; + return VDO_SUCCESS; +} + +void vdo_memory_init(void) +{ + spin_lock_init(&memory_stats.lock); + vdo_initialize_thread_registry(&allocating_threads); +} + +void vdo_memory_exit(void) +{ + VDO_ASSERT_LOG_ONLY(memory_stats.kmalloc_bytes == 0, + "kmalloc memory used (%zd bytes in %zd blocks) is returned to the kernel", + memory_stats.kmalloc_bytes, memory_stats.kmalloc_blocks); + VDO_ASSERT_LOG_ONLY(memory_stats.vmalloc_bytes == 0, + "vmalloc memory used (%zd bytes in %zd blocks) is returned to the kernel", + memory_stats.vmalloc_bytes, memory_stats.vmalloc_blocks); + vdo_log_debug("peak usage %zd bytes", memory_stats.peak_bytes); +} + +void vdo_get_memory_stats(u64 *bytes_used, u64 *peak_bytes_used) +{ + unsigned long flags; + + spin_lock_irqsave(&memory_stats.lock, flags); + *bytes_used = memory_stats.kmalloc_bytes + memory_stats.vmalloc_bytes; + *peak_bytes_used = memory_stats.peak_bytes; + spin_unlock_irqrestore(&memory_stats.lock, flags); +} + +/* + * Report stats on any allocated memory that we're tracking. Not all allocation types are + * guaranteed to be tracked in bytes (e.g., bios). + */ +void vdo_report_memory_usage(void) +{ + unsigned long flags; + u64 kmalloc_blocks; + u64 kmalloc_bytes; + u64 vmalloc_blocks; + u64 vmalloc_bytes; + u64 peak_usage; + u64 total_bytes; + + spin_lock_irqsave(&memory_stats.lock, flags); + kmalloc_blocks = memory_stats.kmalloc_blocks; + kmalloc_bytes = memory_stats.kmalloc_bytes; + vmalloc_blocks = memory_stats.vmalloc_blocks; + vmalloc_bytes = memory_stats.vmalloc_bytes; + peak_usage = memory_stats.peak_bytes; + spin_unlock_irqrestore(&memory_stats.lock, flags); + total_bytes = kmalloc_bytes + vmalloc_bytes; + vdo_log_info("current module memory tracking (actual allocation sizes, not requested):"); + vdo_log_info(" %llu bytes in %llu kmalloc blocks", + (unsigned long long) kmalloc_bytes, + (unsigned long long) kmalloc_blocks); + vdo_log_info(" %llu bytes in %llu vmalloc blocks", + (unsigned long long) vmalloc_bytes, + (unsigned long long) vmalloc_blocks); + vdo_log_info(" total %llu bytes, peak usage %llu bytes", + (unsigned long long) total_bytes, (unsigned long long) peak_usage); +} diff --git a/drivers/md/dm-vdo/memory-alloc.h b/drivers/md/dm-vdo/memory-alloc.h new file mode 100644 index 000000000000..0093d9f940d9 --- /dev/null +++ b/drivers/md/dm-vdo/memory-alloc.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_MEMORY_ALLOC_H +#define VDO_MEMORY_ALLOC_H + +#include +#include /* for PAGE_SIZE */ + +#include "permassert.h" +#include "thread-registry.h" + +/* Custom memory allocation function that tracks memory usage */ +int __must_check vdo_allocate_memory(size_t size, size_t align, const char *what, void *ptr); + +/* + * Allocate storage based on element counts, sizes, and alignment. + * + * This is a generalized form of our allocation use case: It allocates an array of objects, + * optionally preceded by one object of another type (i.e., a struct with trailing variable-length + * array), with the alignment indicated. + * + * Why is this inline? The sizes and alignment will always be constant, when invoked through the + * macros below, and often the count will be a compile-time constant 1 or the number of extra bytes + * will be a compile-time constant 0. So at least some of the arithmetic can usually be optimized + * away, and the run-time selection between allocation functions always can. In many cases, it'll + * boil down to just a function call with a constant size. + * + * @count: The number of objects to allocate + * @size: The size of an object + * @extra: The number of additional bytes to allocate + * @align: The required alignment + * @what: What is being allocated (for error logging) + * @ptr: A pointer to hold the allocated memory + * + * Return: VDO_SUCCESS or an error code + */ +static inline int __vdo_do_allocation(size_t count, size_t size, size_t extra, + size_t align, const char *what, void *ptr) +{ + size_t total_size = count * size + extra; + + /* Overflow check: */ + if ((size > 0) && (count > ((SIZE_MAX - extra) / size))) { + /* + * This is kind of a hack: We rely on the fact that SIZE_MAX would cover the entire + * address space (minus one byte) and thus the system can never allocate that much + * and the call will always fail. So we can report an overflow as "out of memory" + * by asking for "merely" SIZE_MAX bytes. + */ + total_size = SIZE_MAX; + } + + return vdo_allocate_memory(total_size, align, what, ptr); +} + +/* + * Allocate one or more elements of the indicated type, logging an error if the allocation fails. + * The memory will be zeroed. + * + * @COUNT: The number of objects to allocate + * @TYPE: The type of objects to allocate. This type determines the alignment of the allocation. + * @WHAT: What is being allocated (for error logging) + * @PTR: A pointer to hold the allocated memory + * + * Return: VDO_SUCCESS or an error code + */ +#define vdo_allocate(COUNT, TYPE, WHAT, PTR) \ + __vdo_do_allocation(COUNT, sizeof(TYPE), 0, __alignof__(TYPE), WHAT, PTR) + +/* + * Allocate one object of an indicated type, followed by one or more elements of a second type, + * logging an error if the allocation fails. The memory will be zeroed. + * + * @TYPE1: The type of the primary object to allocate. This type determines the alignment of the + * allocated memory. + * @COUNT: The number of objects to allocate + * @TYPE2: The type of array objects to allocate + * @WHAT: What is being allocated (for error logging) + * @PTR: A pointer to hold the allocated memory + * + * Return: VDO_SUCCESS or an error code + */ +#define vdo_allocate_extended(TYPE1, COUNT, TYPE2, WHAT, PTR) \ + __extension__({ \ + int _result; \ + TYPE1 **_ptr = (PTR); \ + BUILD_BUG_ON(__alignof__(TYPE1) < __alignof__(TYPE2)); \ + _result = __vdo_do_allocation(COUNT, \ + sizeof(TYPE2), \ + sizeof(TYPE1), \ + __alignof__(TYPE1), \ + WHAT, \ + _ptr); \ + _result; \ + }) + +/* + * Allocate memory starting on a cache line boundary, logging an error if the allocation fails. The + * memory will be zeroed. + * + * @size: The number of bytes to allocate + * @what: What is being allocated (for error logging) + * @ptr: A pointer to hold the allocated memory + * + * Return: VDO_SUCCESS or an error code + */ +static inline int __must_check vdo_allocate_cache_aligned(size_t size, const char *what, void *ptr) +{ + return vdo_allocate_memory(size, L1_CACHE_BYTES, what, ptr); +} + +/* + * Allocate one element of the indicated type immediately, failing if the required memory is not + * immediately available. + * + * @size: The number of bytes to allocate + * @what: What is being allocated (for error logging) + * + * Return: pointer to the memory, or NULL if the memory is not available. + */ +void *__must_check vdo_allocate_memory_nowait(size_t size, const char *what); + +int __must_check vdo_reallocate_memory(void *ptr, size_t old_size, size_t size, + const char *what, void *new_ptr); + +int __must_check vdo_duplicate_string(const char *string, const char *what, + char **new_string); + +/* Free memory allocated with vdo_allocate(). */ +void vdo_free(void *ptr); + +static inline void *__vdo_forget(void **ptr_ptr) +{ + void *ptr = *ptr_ptr; + + *ptr_ptr = NULL; + return ptr; +} + +/* + * Null out a pointer and return a copy to it. This macro should be used when passing a pointer to + * a function for which it is not safe to access the pointer once the function returns. + */ +#define vdo_forget(ptr) __vdo_forget((void **) &(ptr)) + +void vdo_memory_init(void); + +void vdo_memory_exit(void); + +void vdo_register_allocating_thread(struct registered_thread *new_thread, + const bool *flag_ptr); + +void vdo_unregister_allocating_thread(void); + +void vdo_get_memory_stats(u64 *bytes_used, u64 *peak_bytes_used); + +void vdo_report_memory_usage(void); + +#endif /* VDO_MEMORY_ALLOC_H */ diff --git a/drivers/md/dm-vdo/message-stats.c b/drivers/md/dm-vdo/message-stats.c new file mode 100644 index 000000000000..2802cf92922b --- /dev/null +++ b/drivers/md/dm-vdo/message-stats.c @@ -0,0 +1,432 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "dedupe.h" +#include "logger.h" +#include "memory-alloc.h" +#include "message-stats.h" +#include "statistics.h" +#include "thread-device.h" +#include "vdo.h" + +static void write_u64(char *prefix, u64 value, char *suffix, char **buf, + unsigned int *maxlen) +{ + int count; + + count = scnprintf(*buf, *maxlen, "%s%llu%s", prefix == NULL ? "" : prefix, + value, suffix == NULL ? "" : suffix); + *buf += count; + *maxlen -= count; +} + +static void write_u32(char *prefix, u32 value, char *suffix, char **buf, + unsigned int *maxlen) +{ + int count; + + count = scnprintf(*buf, *maxlen, "%s%u%s", prefix == NULL ? "" : prefix, + value, suffix == NULL ? "" : suffix); + *buf += count; + *maxlen -= count; +} + +static void write_block_count_t(char *prefix, block_count_t value, char *suffix, + char **buf, unsigned int *maxlen) +{ + int count; + + count = scnprintf(*buf, *maxlen, "%s%llu%s", prefix == NULL ? "" : prefix, + value, suffix == NULL ? "" : suffix); + *buf += count; + *maxlen -= count; +} + +static void write_string(char *prefix, char *value, char *suffix, char **buf, + unsigned int *maxlen) +{ + int count; + + count = scnprintf(*buf, *maxlen, "%s%s%s", prefix == NULL ? "" : prefix, + value, suffix == NULL ? "" : suffix); + *buf += count; + *maxlen -= count; +} + +static void write_bool(char *prefix, bool value, char *suffix, char **buf, + unsigned int *maxlen) +{ + int count; + + count = scnprintf(*buf, *maxlen, "%s%d%s", prefix == NULL ? "" : prefix, + value, suffix == NULL ? "" : suffix); + *buf += count; + *maxlen -= count; +} + +static void write_u8(char *prefix, u8 value, char *suffix, char **buf, + unsigned int *maxlen) +{ + int count; + + count = scnprintf(*buf, *maxlen, "%s%u%s", prefix == NULL ? "" : prefix, + value, suffix == NULL ? "" : suffix); + *buf += count; + *maxlen -= count; +} + +static void write_block_allocator_statistics(char *prefix, + struct block_allocator_statistics *stats, + char *suffix, char **buf, + unsigned int *maxlen) +{ + write_string(prefix, "{ ", NULL, buf, maxlen); + /* The total number of slabs from which blocks may be allocated */ + write_u64("slabCount : ", stats->slab_count, ", ", buf, maxlen); + /* The total number of slabs from which blocks have ever been allocated */ + write_u64("slabsOpened : ", stats->slabs_opened, ", ", buf, maxlen); + /* The number of times since loading that a slab has been re-opened */ + write_u64("slabsReopened : ", stats->slabs_reopened, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); +} + +static void write_commit_statistics(char *prefix, struct commit_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) +{ + write_string(prefix, "{ ", NULL, buf, maxlen); + /* The total number of items on which processing has started */ + write_u64("started : ", stats->started, ", ", buf, maxlen); + /* The total number of items for which a write operation has been issued */ + write_u64("written : ", stats->written, ", ", buf, maxlen); + /* The total number of items for which a write operation has completed */ + write_u64("committed : ", stats->committed, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); +} + +static void write_recovery_journal_statistics(char *prefix, + struct recovery_journal_statistics *stats, + char *suffix, char **buf, + unsigned int *maxlen) +{ + write_string(prefix, "{ ", NULL, buf, maxlen); + /* Number of times the on-disk journal was full */ + write_u64("diskFull : ", stats->disk_full, ", ", buf, maxlen); + /* Number of times the recovery journal requested slab journal commits. */ + write_u64("slabJournalCommitsRequested : ", + stats->slab_journal_commits_requested, ", ", buf, maxlen); + /* Write/Commit totals for individual journal entries */ + write_commit_statistics("entries : ", &stats->entries, ", ", buf, maxlen); + /* Write/Commit totals for journal blocks */ + write_commit_statistics("blocks : ", &stats->blocks, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); +} + +static void write_packer_statistics(char *prefix, struct packer_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) +{ + write_string(prefix, "{ ", NULL, buf, maxlen); + /* Number of compressed data items written since startup */ + write_u64("compressedFragmentsWritten : ", + stats->compressed_fragments_written, ", ", buf, maxlen); + /* Number of blocks containing compressed items written since startup */ + write_u64("compressedBlocksWritten : ", + stats->compressed_blocks_written, ", ", buf, maxlen); + /* Number of VIOs that are pending in the packer */ + write_u64("compressedFragmentsInPacker : ", + stats->compressed_fragments_in_packer, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); +} + +static void write_slab_journal_statistics(char *prefix, + struct slab_journal_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) +{ + write_string(prefix, "{ ", NULL, buf, maxlen); + /* Number of times the on-disk journal was full */ + write_u64("diskFullCount : ", stats->disk_full_count, ", ", buf, maxlen); + /* Number of times an entry was added over the flush threshold */ + write_u64("flushCount : ", stats->flush_count, ", ", buf, maxlen); + /* Number of times an entry was added over the block threshold */ + write_u64("blockedCount : ", stats->blocked_count, ", ", buf, maxlen); + /* Number of times a tail block was written */ + write_u64("blocksWritten : ", stats->blocks_written, ", ", buf, maxlen); + /* Number of times we had to wait for the tail to write */ + write_u64("tailBusyCount : ", stats->tail_busy_count, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); +} + +static void write_slab_summary_statistics(char *prefix, + struct slab_summary_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) +{ + write_string(prefix, "{ ", NULL, buf, maxlen); + /* Number of blocks written */ + write_u64("blocksWritten : ", stats->blocks_written, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); +} + +static void write_ref_counts_statistics(char *prefix, struct ref_counts_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) +{ + write_string(prefix, "{ ", NULL, buf, maxlen); + /* Number of reference blocks written */ + write_u64("blocksWritten : ", stats->blocks_written, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); +} + +static void write_block_map_statistics(char *prefix, struct block_map_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) +{ + write_string(prefix, "{ ", NULL, buf, maxlen); + /* number of dirty (resident) pages */ + write_u32("dirtyPages : ", stats->dirty_pages, ", ", buf, maxlen); + /* number of clean (resident) pages */ + write_u32("cleanPages : ", stats->clean_pages, ", ", buf, maxlen); + /* number of free pages */ + write_u32("freePages : ", stats->free_pages, ", ", buf, maxlen); + /* number of pages in failed state */ + write_u32("failedPages : ", stats->failed_pages, ", ", buf, maxlen); + /* number of pages incoming */ + write_u32("incomingPages : ", stats->incoming_pages, ", ", buf, maxlen); + /* number of pages outgoing */ + write_u32("outgoingPages : ", stats->outgoing_pages, ", ", buf, maxlen); + /* how many times free page not avail */ + write_u32("cachePressure : ", stats->cache_pressure, ", ", buf, maxlen); + /* number of get_vdo_page() calls for read */ + write_u64("readCount : ", stats->read_count, ", ", buf, maxlen); + /* number of get_vdo_page() calls for write */ + write_u64("writeCount : ", stats->write_count, ", ", buf, maxlen); + /* number of times pages failed to read */ + write_u64("failedReads : ", stats->failed_reads, ", ", buf, maxlen); + /* number of times pages failed to write */ + write_u64("failedWrites : ", stats->failed_writes, ", ", buf, maxlen); + /* number of gets that are reclaimed */ + write_u64("reclaimed : ", stats->reclaimed, ", ", buf, maxlen); + /* number of gets for outgoing pages */ + write_u64("readOutgoing : ", stats->read_outgoing, ", ", buf, maxlen); + /* number of gets that were already there */ + write_u64("foundInCache : ", stats->found_in_cache, ", ", buf, maxlen); + /* number of gets requiring discard */ + write_u64("discardRequired : ", stats->discard_required, ", ", buf, maxlen); + /* number of gets enqueued for their page */ + write_u64("waitForPage : ", stats->wait_for_page, ", ", buf, maxlen); + /* number of gets that have to fetch */ + write_u64("fetchRequired : ", stats->fetch_required, ", ", buf, maxlen); + /* number of page fetches */ + write_u64("pagesLoaded : ", stats->pages_loaded, ", ", buf, maxlen); + /* number of page saves */ + write_u64("pagesSaved : ", stats->pages_saved, ", ", buf, maxlen); + /* the number of flushes issued */ + write_u64("flushCount : ", stats->flush_count, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); +} + +static void write_hash_lock_statistics(char *prefix, struct hash_lock_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) +{ + write_string(prefix, "{ ", NULL, buf, maxlen); + /* Number of times the UDS advice proved correct */ + write_u64("dedupeAdviceValid : ", stats->dedupe_advice_valid, ", ", buf, maxlen); + /* Number of times the UDS advice proved incorrect */ + write_u64("dedupeAdviceStale : ", stats->dedupe_advice_stale, ", ", buf, maxlen); + /* Number of writes with the same data as another in-flight write */ + write_u64("concurrentDataMatches : ", stats->concurrent_data_matches, + ", ", buf, maxlen); + /* Number of writes whose hash collided with an in-flight write */ + write_u64("concurrentHashCollisions : ", + stats->concurrent_hash_collisions, ", ", buf, maxlen); + /* Current number of dedupe queries that are in flight */ + write_u32("currDedupeQueries : ", stats->curr_dedupe_queries, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); +} + +static void write_error_statistics(char *prefix, struct error_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) +{ + write_string(prefix, "{ ", NULL, buf, maxlen); + /* number of times VDO got an invalid dedupe advice PBN from UDS */ + write_u64("invalidAdvicePBNCount : ", stats->invalid_advice_pbn_count, + ", ", buf, maxlen); + /* number of times a VIO completed with a VDO_NO_SPACE error */ + write_u64("noSpaceErrorCount : ", stats->no_space_error_count, ", ", + buf, maxlen); + /* number of times a VIO completed with a VDO_READ_ONLY error */ + write_u64("readOnlyErrorCount : ", stats->read_only_error_count, ", ", + buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); +} + +static void write_bio_stats(char *prefix, struct bio_stats *stats, char *suffix, + char **buf, unsigned int *maxlen) +{ + write_string(prefix, "{ ", NULL, buf, maxlen); + /* Number of REQ_OP_READ bios */ + write_u64("read : ", stats->read, ", ", buf, maxlen); + /* Number of REQ_OP_WRITE bios with data */ + write_u64("write : ", stats->write, ", ", buf, maxlen); + /* Number of bios tagged with REQ_PREFLUSH and containing no data */ + write_u64("emptyFlush : ", stats->empty_flush, ", ", buf, maxlen); + /* Number of REQ_OP_DISCARD bios */ + write_u64("discard : ", stats->discard, ", ", buf, maxlen); + /* Number of bios tagged with REQ_PREFLUSH */ + write_u64("flush : ", stats->flush, ", ", buf, maxlen); + /* Number of bios tagged with REQ_FUA */ + write_u64("fua : ", stats->fua, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); +} + +static void write_memory_usage(char *prefix, struct memory_usage *stats, char *suffix, + char **buf, unsigned int *maxlen) +{ + write_string(prefix, "{ ", NULL, buf, maxlen); + /* Tracked bytes currently allocated. */ + write_u64("bytesUsed : ", stats->bytes_used, ", ", buf, maxlen); + /* Maximum tracked bytes allocated. */ + write_u64("peakBytesUsed : ", stats->peak_bytes_used, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); +} + +static void write_index_statistics(char *prefix, struct index_statistics *stats, + char *suffix, char **buf, unsigned int *maxlen) +{ + write_string(prefix, "{ ", NULL, buf, maxlen); + /* Number of records stored in the index */ + write_u64("entriesIndexed : ", stats->entries_indexed, ", ", buf, maxlen); + /* Number of post calls that found an existing entry */ + write_u64("postsFound : ", stats->posts_found, ", ", buf, maxlen); + /* Number of post calls that added a new entry */ + write_u64("postsNotFound : ", stats->posts_not_found, ", ", buf, maxlen); + /* Number of query calls that found an existing entry */ + write_u64("queriesFound : ", stats->queries_found, ", ", buf, maxlen); + /* Number of query calls that added a new entry */ + write_u64("queriesNotFound : ", stats->queries_not_found, ", ", buf, maxlen); + /* Number of update calls that found an existing entry */ + write_u64("updatesFound : ", stats->updates_found, ", ", buf, maxlen); + /* Number of update calls that added a new entry */ + write_u64("updatesNotFound : ", stats->updates_not_found, ", ", buf, maxlen); + /* Number of entries discarded */ + write_u64("entriesDiscarded : ", stats->entries_discarded, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); +} + +static void write_vdo_statistics(char *prefix, struct vdo_statistics *stats, char *suffix, + char **buf, unsigned int *maxlen) +{ + write_string(prefix, "{ ", NULL, buf, maxlen); + write_u32("version : ", stats->version, ", ", buf, maxlen); + /* Number of blocks used for data */ + write_u64("dataBlocksUsed : ", stats->data_blocks_used, ", ", buf, maxlen); + /* Number of blocks used for VDO metadata */ + write_u64("overheadBlocksUsed : ", stats->overhead_blocks_used, ", ", + buf, maxlen); + /* Number of logical blocks that are currently mapped to physical blocks */ + write_u64("logicalBlocksUsed : ", stats->logical_blocks_used, ", ", buf, maxlen); + /* number of physical blocks */ + write_block_count_t("physicalBlocks : ", stats->physical_blocks, ", ", + buf, maxlen); + /* number of logical blocks */ + write_block_count_t("logicalBlocks : ", stats->logical_blocks, ", ", + buf, maxlen); + /* Size of the block map page cache, in bytes */ + write_u64("blockMapCacheSize : ", stats->block_map_cache_size, ", ", + buf, maxlen); + /* The physical block size */ + write_u64("blockSize : ", stats->block_size, ", ", buf, maxlen); + /* Number of times the VDO has successfully recovered */ + write_u64("completeRecoveries : ", stats->complete_recoveries, ", ", + buf, maxlen); + /* Number of times the VDO has recovered from read-only mode */ + write_u64("readOnlyRecoveries : ", stats->read_only_recoveries, ", ", + buf, maxlen); + /* String describing the operating mode of the VDO */ + write_string("mode : ", stats->mode, ", ", buf, maxlen); + /* Whether the VDO is in recovery mode */ + write_bool("inRecoveryMode : ", stats->in_recovery_mode, ", ", buf, maxlen); + /* What percentage of recovery mode work has been completed */ + write_u8("recoveryPercentage : ", stats->recovery_percentage, ", ", buf, maxlen); + /* The statistics for the compressed block packer */ + write_packer_statistics("packer : ", &stats->packer, ", ", buf, maxlen); + /* Counters for events in the block allocator */ + write_block_allocator_statistics("allocator : ", &stats->allocator, + ", ", buf, maxlen); + /* Counters for events in the recovery journal */ + write_recovery_journal_statistics("journal : ", &stats->journal, ", ", + buf, maxlen); + /* The statistics for the slab journals */ + write_slab_journal_statistics("slabJournal : ", &stats->slab_journal, + ", ", buf, maxlen); + /* The statistics for the slab summary */ + write_slab_summary_statistics("slabSummary : ", &stats->slab_summary, + ", ", buf, maxlen); + /* The statistics for the reference counts */ + write_ref_counts_statistics("refCounts : ", &stats->ref_counts, ", ", + buf, maxlen); + /* The statistics for the block map */ + write_block_map_statistics("blockMap : ", &stats->block_map, ", ", buf, maxlen); + /* The dedupe statistics from hash locks */ + write_hash_lock_statistics("hashLock : ", &stats->hash_lock, ", ", buf, maxlen); + /* Counts of error conditions */ + write_error_statistics("errors : ", &stats->errors, ", ", buf, maxlen); + /* The VDO instance */ + write_u32("instance : ", stats->instance, ", ", buf, maxlen); + /* Current number of active VIOs */ + write_u32("currentVIOsInProgress : ", stats->current_vios_in_progress, + ", ", buf, maxlen); + /* Maximum number of active VIOs */ + write_u32("maxVIOs : ", stats->max_vios, ", ", buf, maxlen); + /* Number of times the UDS index was too slow in responding */ + write_u64("dedupeAdviceTimeouts : ", stats->dedupe_advice_timeouts, + ", ", buf, maxlen); + /* Number of flush requests submitted to the storage device */ + write_u64("flushOut : ", stats->flush_out, ", ", buf, maxlen); + /* Logical block size */ + write_u64("logicalBlockSize : ", stats->logical_block_size, ", ", buf, maxlen); + /* Bios submitted into VDO from above */ + write_bio_stats("biosIn : ", &stats->bios_in, ", ", buf, maxlen); + write_bio_stats("biosInPartial : ", &stats->bios_in_partial, ", ", buf, maxlen); + /* Bios submitted onward for user data */ + write_bio_stats("biosOut : ", &stats->bios_out, ", ", buf, maxlen); + /* Bios submitted onward for metadata */ + write_bio_stats("biosMeta : ", &stats->bios_meta, ", ", buf, maxlen); + write_bio_stats("biosJournal : ", &stats->bios_journal, ", ", buf, maxlen); + write_bio_stats("biosPageCache : ", &stats->bios_page_cache, ", ", buf, maxlen); + write_bio_stats("biosOutCompleted : ", &stats->bios_out_completed, ", ", + buf, maxlen); + write_bio_stats("biosMetaCompleted : ", &stats->bios_meta_completed, + ", ", buf, maxlen); + write_bio_stats("biosJournalCompleted : ", + &stats->bios_journal_completed, ", ", buf, maxlen); + write_bio_stats("biosPageCacheCompleted : ", + &stats->bios_page_cache_completed, ", ", buf, maxlen); + write_bio_stats("biosAcknowledged : ", &stats->bios_acknowledged, ", ", + buf, maxlen); + write_bio_stats("biosAcknowledgedPartial : ", + &stats->bios_acknowledged_partial, ", ", buf, maxlen); + /* Current number of bios in progress */ + write_bio_stats("biosInProgress : ", &stats->bios_in_progress, ", ", + buf, maxlen); + /* Memory usage stats. */ + write_memory_usage("memoryUsage : ", &stats->memory_usage, ", ", buf, maxlen); + /* The statistics for the UDS index */ + write_index_statistics("index : ", &stats->index, ", ", buf, maxlen); + write_string(NULL, "}", suffix, buf, maxlen); +} + +int vdo_write_stats(struct vdo *vdo, char *buf, unsigned int maxlen) +{ + struct vdo_statistics *stats; + int result; + + result = vdo_allocate(1, struct vdo_statistics, __func__, &stats); + if (result != VDO_SUCCESS) { + vdo_log_error("Cannot allocate memory to write VDO statistics"); + return result; + } + + vdo_fetch_statistics(vdo, stats); + write_vdo_statistics(NULL, stats, NULL, &buf, &maxlen); + vdo_free(stats); + return VDO_SUCCESS; +} diff --git a/drivers/md/dm-vdo/message-stats.h b/drivers/md/dm-vdo/message-stats.h new file mode 100644 index 000000000000..f7fceca9acab --- /dev/null +++ b/drivers/md/dm-vdo/message-stats.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_MESSAGE_STATS_H +#define VDO_MESSAGE_STATS_H + +#include "types.h" + +int vdo_write_stats(struct vdo *vdo, char *buf, unsigned int maxlen); + +#endif /* VDO_MESSAGE_STATS_H */ diff --git a/drivers/md/dm-vdo/murmurhash3.c b/drivers/md/dm-vdo/murmurhash3.c new file mode 100644 index 000000000000..00c9b9c05001 --- /dev/null +++ b/drivers/md/dm-vdo/murmurhash3.c @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: LGPL-2.1+ +/* + * MurmurHash3 was written by Austin Appleby, and is placed in the public + * domain. The author hereby disclaims copyright to this source code. + * + * Adapted by John Wiele (jwiele@redhat.com). + */ + +#include "murmurhash3.h" + +static inline u64 rotl64(u64 x, s8 r) +{ + return (x << r) | (x >> (64 - r)); +} + +#define ROTL64(x, y) rotl64(x, y) +static __always_inline u64 getblock64(const u64 *p, int i) +{ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return p[i]; +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return __builtin_bswap64(p[i]); +#else +#error "can't figure out byte order" +#endif +} + +static __always_inline void putblock64(u64 *p, int i, u64 value) +{ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + p[i] = value; +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + p[i] = __builtin_bswap64(value); +#else +#error "can't figure out byte order" +#endif +} + +/* Finalization mix - force all bits of a hash block to avalanche */ + +static __always_inline u64 fmix64(u64 k) +{ + k ^= k >> 33; + k *= 0xff51afd7ed558ccdLLU; + k ^= k >> 33; + k *= 0xc4ceb9fe1a85ec53LLU; + k ^= k >> 33; + + return k; +} + +void murmurhash3_128(const void *key, const int len, const u32 seed, void *out) +{ + const u8 *data = key; + const int nblocks = len / 16; + + u64 h1 = seed; + u64 h2 = seed; + + const u64 c1 = 0x87c37b91114253d5LLU; + const u64 c2 = 0x4cf5ad432745937fLLU; + + /* body */ + + const u64 *blocks = (const u64 *)(data); + + int i; + + for (i = 0; i < nblocks; i++) { + u64 k1 = getblock64(blocks, i * 2 + 0); + u64 k2 = getblock64(blocks, i * 2 + 1); + + k1 *= c1; + k1 = ROTL64(k1, 31); + k1 *= c2; + h1 ^= k1; + + h1 = ROTL64(h1, 27); + h1 += h2; + h1 = h1 * 5 + 0x52dce729; + + k2 *= c2; + k2 = ROTL64(k2, 33); + k2 *= c1; + h2 ^= k2; + + h2 = ROTL64(h2, 31); + h2 += h1; + h2 = h2 * 5 + 0x38495ab5; + } + + /* tail */ + + { + const u8 *tail = (const u8 *)(data + nblocks * 16); + + u64 k1 = 0; + u64 k2 = 0; + + switch (len & 15) { + case 15: + k2 ^= ((u64)tail[14]) << 48; + fallthrough; + case 14: + k2 ^= ((u64)tail[13]) << 40; + fallthrough; + case 13: + k2 ^= ((u64)tail[12]) << 32; + fallthrough; + case 12: + k2 ^= ((u64)tail[11]) << 24; + fallthrough; + case 11: + k2 ^= ((u64)tail[10]) << 16; + fallthrough; + case 10: + k2 ^= ((u64)tail[9]) << 8; + fallthrough; + case 9: + k2 ^= ((u64)tail[8]) << 0; + k2 *= c2; + k2 = ROTL64(k2, 33); + k2 *= c1; + h2 ^= k2; + fallthrough; + + case 8: + k1 ^= ((u64)tail[7]) << 56; + fallthrough; + case 7: + k1 ^= ((u64)tail[6]) << 48; + fallthrough; + case 6: + k1 ^= ((u64)tail[5]) << 40; + fallthrough; + case 5: + k1 ^= ((u64)tail[4]) << 32; + fallthrough; + case 4: + k1 ^= ((u64)tail[3]) << 24; + fallthrough; + case 3: + k1 ^= ((u64)tail[2]) << 16; + fallthrough; + case 2: + k1 ^= ((u64)tail[1]) << 8; + fallthrough; + case 1: + k1 ^= ((u64)tail[0]) << 0; + k1 *= c1; + k1 = ROTL64(k1, 31); + k1 *= c2; + h1 ^= k1; + break; + default: + break; + }; + } + /* finalization */ + + h1 ^= len; + h2 ^= len; + + h1 += h2; + h2 += h1; + + h1 = fmix64(h1); + h2 = fmix64(h2); + + h1 += h2; + h2 += h1; + + putblock64((u64 *)out, 0, h1); + putblock64((u64 *)out, 1, h2); +} diff --git a/drivers/md/dm-vdo/murmurhash3.h b/drivers/md/dm-vdo/murmurhash3.h new file mode 100644 index 000000000000..d84711ddb659 --- /dev/null +++ b/drivers/md/dm-vdo/murmurhash3.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +/* + * MurmurHash3 was written by Austin Appleby, and is placed in the public + * domain. The author hereby disclaims copyright to this source code. + */ + +#ifndef _MURMURHASH3_H_ +#define _MURMURHASH3_H_ + +#include +#include + +void murmurhash3_128(const void *key, int len, u32 seed, void *out); + +#endif /* _MURMURHASH3_H_ */ diff --git a/drivers/md/dm-vdo/numeric.h b/drivers/md/dm-vdo/numeric.h new file mode 100644 index 000000000000..dc8c400b21d2 --- /dev/null +++ b/drivers/md/dm-vdo/numeric.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_NUMERIC_H +#define UDS_NUMERIC_H + +#include +#include +#include + +/* + * These utilities encode or decode a number from an offset in a larger data buffer and then + * advance the offset pointer to the next field in the buffer. + */ + +static inline void decode_s64_le(const u8 *buffer, size_t *offset, s64 *decoded) +{ + *decoded = get_unaligned_le64(buffer + *offset); + *offset += sizeof(s64); +} + +static inline void encode_s64_le(u8 *data, size_t *offset, s64 to_encode) +{ + put_unaligned_le64(to_encode, data + *offset); + *offset += sizeof(s64); +} + +static inline void decode_u64_le(const u8 *buffer, size_t *offset, u64 *decoded) +{ + *decoded = get_unaligned_le64(buffer + *offset); + *offset += sizeof(u64); +} + +static inline void encode_u64_le(u8 *data, size_t *offset, u64 to_encode) +{ + put_unaligned_le64(to_encode, data + *offset); + *offset += sizeof(u64); +} + +static inline void decode_s32_le(const u8 *buffer, size_t *offset, s32 *decoded) +{ + *decoded = get_unaligned_le32(buffer + *offset); + *offset += sizeof(s32); +} + +static inline void encode_s32_le(u8 *data, size_t *offset, s32 to_encode) +{ + put_unaligned_le32(to_encode, data + *offset); + *offset += sizeof(s32); +} + +static inline void decode_u32_le(const u8 *buffer, size_t *offset, u32 *decoded) +{ + *decoded = get_unaligned_le32(buffer + *offset); + *offset += sizeof(u32); +} + +static inline void encode_u32_le(u8 *data, size_t *offset, u32 to_encode) +{ + put_unaligned_le32(to_encode, data + *offset); + *offset += sizeof(u32); +} + +static inline void decode_u16_le(const u8 *buffer, size_t *offset, u16 *decoded) +{ + *decoded = get_unaligned_le16(buffer + *offset); + *offset += sizeof(u16); +} + +static inline void encode_u16_le(u8 *data, size_t *offset, u16 to_encode) +{ + put_unaligned_le16(to_encode, data + *offset); + *offset += sizeof(u16); +} + +#endif /* UDS_NUMERIC_H */ diff --git a/drivers/md/dm-vdo/packer.c b/drivers/md/dm-vdo/packer.c new file mode 100644 index 000000000000..16cf29b4c90a --- /dev/null +++ b/drivers/md/dm-vdo/packer.c @@ -0,0 +1,780 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "packer.h" + +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" +#include "string-utils.h" + +#include "admin-state.h" +#include "completion.h" +#include "constants.h" +#include "data-vio.h" +#include "dedupe.h" +#include "encodings.h" +#include "io-submitter.h" +#include "physical-zone.h" +#include "status-codes.h" +#include "vdo.h" +#include "vio.h" + +static const struct version_number COMPRESSED_BLOCK_1_0 = { + .major_version = 1, + .minor_version = 0, +}; + +#define COMPRESSED_BLOCK_1_0_SIZE (4 + 4 + (2 * VDO_MAX_COMPRESSION_SLOTS)) + +/** + * vdo_get_compressed_block_fragment() - Get a reference to a compressed fragment from a compressed + * block. + * @mapping_state [in] The mapping state for the look up. + * @compressed_block [in] The compressed block that was read from disk. + * @fragment_offset [out] The offset of the fragment within a compressed block. + * @fragment_size [out] The size of the fragment. + * + * Return: If a valid compressed fragment is found, VDO_SUCCESS; otherwise, VDO_INVALID_FRAGMENT if + * the fragment is invalid. + */ +int vdo_get_compressed_block_fragment(enum block_mapping_state mapping_state, + struct compressed_block *block, + u16 *fragment_offset, u16 *fragment_size) +{ + u16 compressed_size; + u16 offset = 0; + unsigned int i; + u8 slot; + struct version_number version; + + if (!vdo_is_state_compressed(mapping_state)) + return VDO_INVALID_FRAGMENT; + + version = vdo_unpack_version_number(block->header.version); + if (!vdo_are_same_version(version, COMPRESSED_BLOCK_1_0)) + return VDO_INVALID_FRAGMENT; + + slot = mapping_state - VDO_MAPPING_STATE_COMPRESSED_BASE; + if (slot >= VDO_MAX_COMPRESSION_SLOTS) + return VDO_INVALID_FRAGMENT; + + compressed_size = __le16_to_cpu(block->header.sizes[slot]); + for (i = 0; i < slot; i++) { + offset += __le16_to_cpu(block->header.sizes[i]); + if (offset >= VDO_COMPRESSED_BLOCK_DATA_SIZE) + return VDO_INVALID_FRAGMENT; + } + + if ((offset + compressed_size) > VDO_COMPRESSED_BLOCK_DATA_SIZE) + return VDO_INVALID_FRAGMENT; + + *fragment_offset = offset; + *fragment_size = compressed_size; + return VDO_SUCCESS; +} + +/** + * assert_on_packer_thread() - Check that we are on the packer thread. + * @packer: The packer. + * @caller: The function which is asserting. + */ +static inline void assert_on_packer_thread(struct packer *packer, const char *caller) +{ + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == packer->thread_id), + "%s() called from packer thread", caller); +} + +/** + * insert_in_sorted_list() - Insert a bin to the list. + * @packer: The packer. + * @bin: The bin to move to its sorted position. + * + * The list is in ascending order of free space. Since all bins are already in the list, this + * actually moves the bin to the correct position in the list. + */ +static void insert_in_sorted_list(struct packer *packer, struct packer_bin *bin) +{ + struct packer_bin *active_bin; + + list_for_each_entry(active_bin, &packer->bins, list) + if (active_bin->free_space > bin->free_space) { + list_move_tail(&bin->list, &active_bin->list); + return; + } + + list_move_tail(&bin->list, &packer->bins); +} + +/** + * make_bin() - Allocate a bin and put it into the packer's list. + * @packer: The packer. + */ +static int __must_check make_bin(struct packer *packer) +{ + struct packer_bin *bin; + int result; + + result = vdo_allocate_extended(struct packer_bin, VDO_MAX_COMPRESSION_SLOTS, + struct vio *, __func__, &bin); + if (result != VDO_SUCCESS) + return result; + + bin->free_space = VDO_COMPRESSED_BLOCK_DATA_SIZE; + INIT_LIST_HEAD(&bin->list); + list_add_tail(&bin->list, &packer->bins); + return VDO_SUCCESS; +} + +/** + * vdo_make_packer() - Make a new block packer. + * + * @vdo: The vdo to which this packer belongs. + * @bin_count: The number of partial bins to keep in memory. + * @packer_ptr: A pointer to hold the new packer. + * + * Return: VDO_SUCCESS or an error + */ +int vdo_make_packer(struct vdo *vdo, block_count_t bin_count, struct packer **packer_ptr) +{ + struct packer *packer; + block_count_t i; + int result; + + result = vdo_allocate(1, struct packer, __func__, &packer); + if (result != VDO_SUCCESS) + return result; + + packer->thread_id = vdo->thread_config.packer_thread; + packer->size = bin_count; + INIT_LIST_HEAD(&packer->bins); + vdo_set_admin_state_code(&packer->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + + for (i = 0; i < bin_count; i++) { + result = make_bin(packer); + if (result != VDO_SUCCESS) { + vdo_free_packer(packer); + return result; + } + } + + /* + * The canceled bin can hold up to half the number of user vios. Every canceled vio in the + * bin must have a canceler for which it is waiting, and any canceler will only have + * canceled one lock holder at a time. + */ + result = vdo_allocate_extended(struct packer_bin, MAXIMUM_VDO_USER_VIOS / 2, + struct vio *, __func__, &packer->canceled_bin); + if (result != VDO_SUCCESS) { + vdo_free_packer(packer); + return result; + } + + result = vdo_make_default_thread(vdo, packer->thread_id); + if (result != VDO_SUCCESS) { + vdo_free_packer(packer); + return result; + } + + *packer_ptr = packer; + return VDO_SUCCESS; +} + +/** + * vdo_free_packer() - Free a block packer. + * @packer: The packer to free. + */ +void vdo_free_packer(struct packer *packer) +{ + struct packer_bin *bin, *tmp; + + if (packer == NULL) + return; + + list_for_each_entry_safe(bin, tmp, &packer->bins, list) { + list_del_init(&bin->list); + vdo_free(bin); + } + + vdo_free(vdo_forget(packer->canceled_bin)); + vdo_free(packer); +} + +/** + * get_packer_from_data_vio() - Get the packer from a data_vio. + * @data_vio: The data_vio. + * + * Return: The packer from the VDO to which the data_vio belongs. + */ +static inline struct packer *get_packer_from_data_vio(struct data_vio *data_vio) +{ + return vdo_from_data_vio(data_vio)->packer; +} + +/** + * vdo_get_packer_statistics() - Get the current statistics from the packer. + * @packer: The packer to query. + * + * Return: a copy of the current statistics for the packer. + */ +struct packer_statistics vdo_get_packer_statistics(const struct packer *packer) +{ + const struct packer_statistics *stats = &packer->statistics; + + return (struct packer_statistics) { + .compressed_fragments_written = READ_ONCE(stats->compressed_fragments_written), + .compressed_blocks_written = READ_ONCE(stats->compressed_blocks_written), + .compressed_fragments_in_packer = READ_ONCE(stats->compressed_fragments_in_packer), + }; +} + +/** + * abort_packing() - Abort packing a data_vio. + * @data_vio: The data_vio to abort. + */ +static void abort_packing(struct data_vio *data_vio) +{ + struct packer *packer = get_packer_from_data_vio(data_vio); + + WRITE_ONCE(packer->statistics.compressed_fragments_in_packer, + packer->statistics.compressed_fragments_in_packer - 1); + + write_data_vio(data_vio); +} + +/** + * release_compressed_write_waiter() - Update a data_vio for which a successful compressed write + * has completed and send it on its way. + + * @data_vio: The data_vio to release. + * @allocation: The allocation to which the compressed block was written. + */ +static void release_compressed_write_waiter(struct data_vio *data_vio, + struct allocation *allocation) +{ + data_vio->new_mapped = (struct zoned_pbn) { + .pbn = allocation->pbn, + .zone = allocation->zone, + .state = data_vio->compression.slot + VDO_MAPPING_STATE_COMPRESSED_BASE, + }; + + vdo_share_compressed_write_lock(data_vio, allocation->lock); + update_metadata_for_data_vio_write(data_vio, allocation->lock); +} + +/** + * finish_compressed_write() - Finish a compressed block write. + * @completion: The compressed write completion. + * + * This callback is registered in continue_after_allocation(). + */ +static void finish_compressed_write(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + struct data_vio *client, *next; + + assert_data_vio_in_allocated_zone(agent); + + /* + * Process all the non-agent waiters first to ensure that the pbn lock can not be released + * until all of them have had a chance to journal their increfs. + */ + for (client = agent->compression.next_in_batch; client != NULL; client = next) { + next = client->compression.next_in_batch; + release_compressed_write_waiter(client, &agent->allocation); + } + + completion->error_handler = handle_data_vio_error; + release_compressed_write_waiter(agent, &agent->allocation); +} + +static void handle_compressed_write_error(struct vdo_completion *completion) +{ + struct data_vio *agent = as_data_vio(completion); + struct allocation *allocation = &agent->allocation; + struct data_vio *client, *next; + + if (vdo_requeue_completion_if_needed(completion, allocation->zone->thread_id)) + return; + + update_vio_error_stats(as_vio(completion), + "Completing compressed write vio for physical block %llu with error", + (unsigned long long) allocation->pbn); + + for (client = agent->compression.next_in_batch; client != NULL; client = next) { + next = client->compression.next_in_batch; + write_data_vio(client); + } + + /* Now that we've released the batch from the packer, forget the error and continue on. */ + vdo_reset_completion(completion); + completion->error_handler = handle_data_vio_error; + write_data_vio(agent); +} + +/** + * add_to_bin() - Put a data_vio in a specific packer_bin in which it will definitely fit. + * @bin: The bin in which to put the data_vio. + * @data_vio: The data_vio to add. + */ +static void add_to_bin(struct packer_bin *bin, struct data_vio *data_vio) +{ + data_vio->compression.bin = bin; + data_vio->compression.slot = bin->slots_used; + bin->incoming[bin->slots_used++] = data_vio; +} + +/** + * remove_from_bin() - Get the next data_vio whose compression has not been canceled from a bin. + * @packer: The packer. + * @bin: The bin from which to get a data_vio. + * + * Any canceled data_vios will be moved to the canceled bin. + * Return: An uncanceled data_vio from the bin or NULL if there are none. + */ +static struct data_vio *remove_from_bin(struct packer *packer, struct packer_bin *bin) +{ + while (bin->slots_used > 0) { + struct data_vio *data_vio = bin->incoming[--bin->slots_used]; + + if (!advance_data_vio_compression_stage(data_vio).may_not_compress) { + data_vio->compression.bin = NULL; + return data_vio; + } + + add_to_bin(packer->canceled_bin, data_vio); + } + + /* The bin is now empty. */ + bin->free_space = VDO_COMPRESSED_BLOCK_DATA_SIZE; + return NULL; +} + +/** + * initialize_compressed_block() - Initialize a compressed block. + * @block: The compressed block to initialize. + * @size: The size of the agent's fragment. + * + * This method initializes the compressed block in the compressed write agent. Because the + * compressor already put the agent's compressed fragment at the start of the compressed block's + * data field, it needn't be copied. So all we need do is initialize the header and set the size of + * the agent's fragment. + */ +static void initialize_compressed_block(struct compressed_block *block, u16 size) +{ + /* + * Make sure the block layout isn't accidentally changed by changing the length of the + * block header. + */ + BUILD_BUG_ON(sizeof(struct compressed_block_header) != COMPRESSED_BLOCK_1_0_SIZE); + + block->header.version = vdo_pack_version_number(COMPRESSED_BLOCK_1_0); + block->header.sizes[0] = __cpu_to_le16(size); +} + +/** + * pack_fragment() - Pack a data_vio's fragment into the compressed block in which it is already + * known to fit. + * @compression: The agent's compression_state to pack in to. + * @data_vio: The data_vio to pack. + * @offset: The offset into the compressed block at which to pack the fragment. + * @compressed_block: The compressed block which will be written out when batch is fully packed. + * + * Return: The new amount of space used. + */ +static block_size_t __must_check pack_fragment(struct compression_state *compression, + struct data_vio *data_vio, + block_size_t offset, slot_number_t slot, + struct compressed_block *block) +{ + struct compression_state *to_pack = &data_vio->compression; + char *fragment = to_pack->block->data; + + to_pack->next_in_batch = compression->next_in_batch; + compression->next_in_batch = data_vio; + to_pack->slot = slot; + block->header.sizes[slot] = __cpu_to_le16(to_pack->size); + memcpy(&block->data[offset], fragment, to_pack->size); + return (offset + to_pack->size); +} + +/** + * compressed_write_end_io() - The bio_end_io for a compressed block write. + * @bio: The bio for the compressed write. + */ +static void compressed_write_end_io(struct bio *bio) +{ + struct data_vio *data_vio = vio_as_data_vio(bio->bi_private); + + vdo_count_completed_bios(bio); + set_data_vio_allocated_zone_callback(data_vio, finish_compressed_write); + continue_data_vio_with_error(data_vio, blk_status_to_errno(bio->bi_status)); +} + +/** + * write_bin() - Write out a bin. + * @packer: The packer. + * @bin: The bin to write. + */ +static void write_bin(struct packer *packer, struct packer_bin *bin) +{ + int result; + block_size_t offset; + slot_number_t slot = 1; + struct compression_state *compression; + struct compressed_block *block; + struct data_vio *agent = remove_from_bin(packer, bin); + struct data_vio *client; + struct packer_statistics *stats; + + if (agent == NULL) + return; + + compression = &agent->compression; + compression->slot = 0; + block = compression->block; + initialize_compressed_block(block, compression->size); + offset = compression->size; + + while ((client = remove_from_bin(packer, bin)) != NULL) + offset = pack_fragment(compression, client, offset, slot++, block); + + /* + * If the batch contains only a single vio, then we save nothing by saving the compressed + * form. Continue processing the single vio in the batch. + */ + if (slot == 1) { + abort_packing(agent); + return; + } + + if (slot < VDO_MAX_COMPRESSION_SLOTS) { + /* Clear out the sizes of the unused slots */ + memset(&block->header.sizes[slot], 0, + (VDO_MAX_COMPRESSION_SLOTS - slot) * sizeof(__le16)); + } + + agent->vio.completion.error_handler = handle_compressed_write_error; + if (vdo_is_read_only(vdo_from_data_vio(agent))) { + continue_data_vio_with_error(agent, VDO_READ_ONLY); + return; + } + + result = vio_reset_bio(&agent->vio, (char *) block, compressed_write_end_io, + REQ_OP_WRITE, agent->allocation.pbn); + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(agent, result); + return; + } + + /* + * Once the compressed write is submitted, the fragments are no longer in the packer, so + * update stats now. + */ + stats = &packer->statistics; + WRITE_ONCE(stats->compressed_fragments_in_packer, + (stats->compressed_fragments_in_packer - slot)); + WRITE_ONCE(stats->compressed_fragments_written, + (stats->compressed_fragments_written + slot)); + WRITE_ONCE(stats->compressed_blocks_written, + stats->compressed_blocks_written + 1); + + vdo_submit_data_vio(agent); +} + +/** + * add_data_vio_to_packer_bin() - Add a data_vio to a bin's incoming queue + * @packer: The packer. + * @bin: The bin to which to add the data_vio. + * @data_vio: The data_vio to add to the bin's queue. + * + * Adds a data_vio to a bin's incoming queue, handles logical space change, and calls physical + * space processor. + */ +static void add_data_vio_to_packer_bin(struct packer *packer, struct packer_bin *bin, + struct data_vio *data_vio) +{ + /* If the selected bin doesn't have room, start a new batch to make room. */ + if (bin->free_space < data_vio->compression.size) + write_bin(packer, bin); + + add_to_bin(bin, data_vio); + bin->free_space -= data_vio->compression.size; + + /* If we happen to exactly fill the bin, start a new batch. */ + if ((bin->slots_used == VDO_MAX_COMPRESSION_SLOTS) || + (bin->free_space == 0)) + write_bin(packer, bin); + + /* Now that we've finished changing the free space, restore the sort order. */ + insert_in_sorted_list(packer, bin); +} + +/** + * select_bin() - Select the bin that should be used to pack the compressed data in a data_vio with + * other data_vios. + * @packer: The packer. + * @data_vio: The data_vio. + */ +static struct packer_bin * __must_check select_bin(struct packer *packer, + struct data_vio *data_vio) +{ + /* + * First best fit: select the bin with the least free space that has enough room for the + * compressed data in the data_vio. + */ + struct packer_bin *bin, *fullest_bin; + + list_for_each_entry(bin, &packer->bins, list) { + if (bin->free_space >= data_vio->compression.size) + return bin; + } + + /* + * None of the bins have enough space for the data_vio. We're not allowed to create new + * bins, so we have to overflow one of the existing bins. It's pretty intuitive to select + * the fullest bin, since that "wastes" the least amount of free space in the compressed + * block. But if the space currently used in the fullest bin is smaller than the compressed + * size of the incoming block, it seems wrong to force that bin to write when giving up on + * compressing the incoming data_vio would likewise "waste" the least amount of free space. + */ + fullest_bin = list_first_entry(&packer->bins, struct packer_bin, list); + if (data_vio->compression.size >= + (VDO_COMPRESSED_BLOCK_DATA_SIZE - fullest_bin->free_space)) + return NULL; + + /* + * The fullest bin doesn't have room, but writing it out and starting a new batch with the + * incoming data_vio will increase the packer's free space. + */ + return fullest_bin; +} + +/** + * vdo_attempt_packing() - Attempt to rewrite the data in this data_vio as part of a compressed + * block. + * @data_vio: The data_vio to pack. + */ +void vdo_attempt_packing(struct data_vio *data_vio) +{ + int result; + struct packer_bin *bin; + struct data_vio_compression_status status = get_data_vio_compression_status(data_vio); + struct packer *packer = get_packer_from_data_vio(data_vio); + + assert_on_packer_thread(packer, __func__); + + result = VDO_ASSERT((status.stage == DATA_VIO_COMPRESSING), + "attempt to pack data_vio not ready for packing, stage: %u", + status.stage); + if (result != VDO_SUCCESS) + return; + + /* + * Increment whether or not this data_vio will be packed or not since abort_packing() + * always decrements the counter. + */ + WRITE_ONCE(packer->statistics.compressed_fragments_in_packer, + packer->statistics.compressed_fragments_in_packer + 1); + + /* + * If packing of this data_vio is disallowed for administrative reasons, give up before + * making any state changes. + */ + if (!vdo_is_state_normal(&packer->state) || + (data_vio->flush_generation < packer->flush_generation)) { + abort_packing(data_vio); + return; + } + + /* + * The advance_data_vio_compression_stage() check here verifies that the data_vio is + * allowed to be compressed (if it has already been canceled, we'll fall out here). Once + * the data_vio is in the DATA_VIO_PACKING state, it must be guaranteed to be put in a bin + * before any more requests can be processed by the packer thread. Otherwise, a canceling + * data_vio could attempt to remove the canceled data_vio from the packer and fail to + * rendezvous with it. Thus, we must call select_bin() first to ensure that we will + * actually add the data_vio to a bin before advancing to the DATA_VIO_PACKING stage. + */ + bin = select_bin(packer, data_vio); + if ((bin == NULL) || + (advance_data_vio_compression_stage(data_vio).stage != DATA_VIO_PACKING)) { + abort_packing(data_vio); + return; + } + + add_data_vio_to_packer_bin(packer, bin, data_vio); +} + +/** + * check_for_drain_complete() - Check whether the packer has drained. + * @packer: The packer. + */ +static void check_for_drain_complete(struct packer *packer) +{ + if (vdo_is_state_draining(&packer->state) && (packer->canceled_bin->slots_used == 0)) + vdo_finish_draining(&packer->state); +} + +/** + * write_all_non_empty_bins() - Write out all non-empty bins on behalf of a flush or suspend. + * @packer: The packer being flushed. + */ +static void write_all_non_empty_bins(struct packer *packer) +{ + struct packer_bin *bin; + + list_for_each_entry(bin, &packer->bins, list) + write_bin(packer, bin); + /* + * We don't need to re-sort the bin here since this loop will make every bin have + * the same amount of free space, so every ordering is sorted. + */ + + check_for_drain_complete(packer); +} + +/** + * vdo_flush_packer() - Request that the packer flush asynchronously. + * @packer: The packer to flush. + * + * All bins with at least two compressed data blocks will be written out, and any solitary pending + * VIOs will be released from the packer. While flushing is in progress, any VIOs submitted to + * vdo_attempt_packing() will be continued immediately without attempting to pack them. + */ +void vdo_flush_packer(struct packer *packer) +{ + assert_on_packer_thread(packer, __func__); + if (vdo_is_state_normal(&packer->state)) + write_all_non_empty_bins(packer); +} + +/** + * vdo_remove_lock_holder_from_packer() - Remove a lock holder from the packer. + * @completion: The data_vio which needs a lock held by a data_vio in the packer. The data_vio's + * compression.lock_holder field will point to the data_vio to remove. + */ +void vdo_remove_lock_holder_from_packer(struct vdo_completion *completion) +{ + struct data_vio *data_vio = as_data_vio(completion); + struct packer *packer = get_packer_from_data_vio(data_vio); + struct data_vio *lock_holder; + struct packer_bin *bin; + slot_number_t slot; + + assert_data_vio_in_packer_zone(data_vio); + + lock_holder = vdo_forget(data_vio->compression.lock_holder); + bin = lock_holder->compression.bin; + VDO_ASSERT_LOG_ONLY((bin != NULL), "data_vio in packer has a bin"); + + slot = lock_holder->compression.slot; + bin->slots_used--; + if (slot < bin->slots_used) { + bin->incoming[slot] = bin->incoming[bin->slots_used]; + bin->incoming[slot]->compression.slot = slot; + } + + lock_holder->compression.bin = NULL; + lock_holder->compression.slot = 0; + + if (bin != packer->canceled_bin) { + bin->free_space += lock_holder->compression.size; + insert_in_sorted_list(packer, bin); + } + + abort_packing(lock_holder); + check_for_drain_complete(packer); +} + +/** + * vdo_increment_packer_flush_generation() - Increment the flush generation in the packer. + * @packer: The packer. + * + * This will also cause the packer to flush so that any VIOs from previous generations will exit + * the packer. + */ +void vdo_increment_packer_flush_generation(struct packer *packer) +{ + assert_on_packer_thread(packer, __func__); + packer->flush_generation++; + vdo_flush_packer(packer); +} + +/** + * initiate_drain() - Initiate a drain. + * + * Implements vdo_admin_initiator_fn. + */ +static void initiate_drain(struct admin_state *state) +{ + struct packer *packer = container_of(state, struct packer, state); + + write_all_non_empty_bins(packer); +} + +/** + * vdo_drain_packer() - Drain the packer by preventing any more VIOs from entering the packer and + * then flushing. + * @packer: The packer to drain. + * @completion: The completion to finish when the packer has drained. + */ +void vdo_drain_packer(struct packer *packer, struct vdo_completion *completion) +{ + assert_on_packer_thread(packer, __func__); + vdo_start_draining(&packer->state, VDO_ADMIN_STATE_SUSPENDING, completion, + initiate_drain); +} + +/** + * vdo_resume_packer() - Resume a packer which has been suspended. + * @packer: The packer to resume. + * @parent: The completion to finish when the packer has resumed. + */ +void vdo_resume_packer(struct packer *packer, struct vdo_completion *parent) +{ + assert_on_packer_thread(packer, __func__); + vdo_continue_completion(parent, vdo_resume_if_quiescent(&packer->state)); +} + +static void dump_packer_bin(const struct packer_bin *bin, bool canceled) +{ + if (bin->slots_used == 0) + /* Don't dump empty bins. */ + return; + + vdo_log_info(" %sBin slots_used=%u free_space=%zu", + (canceled ? "Canceled" : ""), bin->slots_used, bin->free_space); + + /* + * FIXME: dump vios in bin->incoming? The vios should have been dumped from the vio pool. + * Maybe just dump their addresses so it's clear they're here? + */ +} + +/** + * vdo_dump_packer() - Dump the packer. + * @packer: The packer. + * + * Context: dumps in a thread-unsafe fashion. + */ +void vdo_dump_packer(const struct packer *packer) +{ + struct packer_bin *bin; + + vdo_log_info("packer"); + vdo_log_info(" flushGeneration=%llu state %s packer_bin_count=%llu", + (unsigned long long) packer->flush_generation, + vdo_get_admin_state_code(&packer->state)->name, + (unsigned long long) packer->size); + + list_for_each_entry(bin, &packer->bins, list) + dump_packer_bin(bin, false); + + dump_packer_bin(packer->canceled_bin, true); +} diff --git a/drivers/md/dm-vdo/packer.h b/drivers/md/dm-vdo/packer.h new file mode 100644 index 000000000000..0f3be44710b5 --- /dev/null +++ b/drivers/md/dm-vdo/packer.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_PACKER_H +#define VDO_PACKER_H + +#include + +#include "admin-state.h" +#include "constants.h" +#include "encodings.h" +#include "statistics.h" +#include "types.h" +#include "wait-queue.h" + +enum { + DEFAULT_PACKER_BINS = 16, +}; + +/* The header of a compressed block. */ +struct compressed_block_header { + /* Unsigned 32-bit major and minor versions, little-endian */ + struct packed_version_number version; + + /* List of unsigned 16-bit compressed block sizes, little-endian */ + __le16 sizes[VDO_MAX_COMPRESSION_SLOTS]; +} __packed; + +enum { + VDO_COMPRESSED_BLOCK_DATA_SIZE = VDO_BLOCK_SIZE - sizeof(struct compressed_block_header), + + /* + * A compressed block is only written if we can pack at least two fragments into it, so a + * fragment which fills the entire data portion of a compressed block is too big. + */ + VDO_MAX_COMPRESSED_FRAGMENT_SIZE = VDO_COMPRESSED_BLOCK_DATA_SIZE - 1, +}; + +/* * The compressed block overlay. */ +struct compressed_block { + struct compressed_block_header header; + char data[VDO_COMPRESSED_BLOCK_DATA_SIZE]; +} __packed; + +/* + * Each packer_bin holds an incomplete batch of data_vios that only partially fill a compressed + * block. The bins are kept in a ring sorted by the amount of unused space so the first bin with + * enough space to hold a newly-compressed data_vio can easily be found. When the bin fills up or + * is flushed, the first uncanceled data_vio in the bin is selected to be the agent for that bin. + * Upon entering the packer, each data_vio already has its compressed data in the first slot of the + * data_vio's compressed_block (overlaid on the data_vio's scratch_block). So the agent's fragment + * is already in place. The fragments for the other uncanceled data_vios in the bin are packed into + * the agent's compressed block. The agent then writes out the compressed block. If the write is + * successful, the agent shares its pbn lock which each of the other data_vios in its compressed + * block and sends each on its way. Finally the agent itself continues on the write path as before. + * + * There is one special bin which is used to hold data_vios which have been canceled and removed + * from their bin by the packer. These data_vios need to wait for the canceller to rendezvous with + * them and so they sit in this special bin. + */ +struct packer_bin { + /* List links for packer.packer_bins */ + struct list_head list; + /* The number of items in the bin */ + slot_number_t slots_used; + /* The number of compressed block bytes remaining in the current batch */ + size_t free_space; + /* The current partial batch of data_vios, waiting for more */ + struct data_vio *incoming[]; +}; + +struct packer { + /* The ID of the packer's callback thread */ + thread_id_t thread_id; + /* The number of bins */ + block_count_t size; + /* A list of all packer_bins, kept sorted by free_space */ + struct list_head bins; + /* + * A bin to hold data_vios which were canceled out of the packer and are waiting to + * rendezvous with the canceling data_vio. + */ + struct packer_bin *canceled_bin; + + /* The current flush generation */ + sequence_number_t flush_generation; + + /* The administrative state of the packer */ + struct admin_state state; + + /* Statistics are only updated on the packer thread, but are accessed from other threads */ + struct packer_statistics statistics; +}; + +int vdo_get_compressed_block_fragment(enum block_mapping_state mapping_state, + struct compressed_block *block, + u16 *fragment_offset, u16 *fragment_size); + +int __must_check vdo_make_packer(struct vdo *vdo, block_count_t bin_count, + struct packer **packer_ptr); + +void vdo_free_packer(struct packer *packer); + +struct packer_statistics __must_check vdo_get_packer_statistics(const struct packer *packer); + +void vdo_attempt_packing(struct data_vio *data_vio); + +void vdo_flush_packer(struct packer *packer); + +void vdo_remove_lock_holder_from_packer(struct vdo_completion *completion); + +void vdo_increment_packer_flush_generation(struct packer *packer); + +void vdo_drain_packer(struct packer *packer, struct vdo_completion *completion); + +void vdo_resume_packer(struct packer *packer, struct vdo_completion *parent); + +void vdo_dump_packer(const struct packer *packer); + +#endif /* VDO_PACKER_H */ diff --git a/drivers/md/dm-vdo/permassert.c b/drivers/md/dm-vdo/permassert.c new file mode 100644 index 000000000000..bf9eccea1cb3 --- /dev/null +++ b/drivers/md/dm-vdo/permassert.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "permassert.h" + +#include "errors.h" +#include "logger.h" + +int vdo_assertion_failed(const char *expression_string, const char *file_name, + int line_number, const char *format, ...) +{ + va_list args; + + va_start(args, format); + + vdo_log_embedded_message(VDO_LOG_ERR, VDO_LOGGING_MODULE_NAME, "assertion \"", + format, args, "\" (%s) failed at %s:%d", + expression_string, file_name, line_number); + vdo_log_backtrace(VDO_LOG_ERR); + + va_end(args); + + return UDS_ASSERTION_FAILED; +} diff --git a/drivers/md/dm-vdo/permassert.h b/drivers/md/dm-vdo/permassert.h new file mode 100644 index 000000000000..c34f2ba650e1 --- /dev/null +++ b/drivers/md/dm-vdo/permassert.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef PERMASSERT_H +#define PERMASSERT_H + +#include + +#include "errors.h" + +/* Utilities for asserting that certain conditions are met */ + +#define STRINGIFY(X) #X + +/* + * A hack to apply the "warn if unused" attribute to an integral expression. + * + * Since GCC doesn't propagate the warn_unused_result attribute to conditional expressions + * incorporating calls to functions with that attribute, this function can be used to wrap such an + * expression. With optimization enabled, this function contributes no additional instructions, but + * the warn_unused_result attribute still applies to the code calling it. + */ +static inline int __must_check vdo_must_use(int value) +{ + return value; +} + +/* Assert that an expression is true and return an error if it is not. */ +#define VDO_ASSERT(expr, ...) vdo_must_use(__VDO_ASSERT(expr, __VA_ARGS__)) + +/* Log a message if the expression is not true. */ +#define VDO_ASSERT_LOG_ONLY(expr, ...) __VDO_ASSERT(expr, __VA_ARGS__) + +#define __VDO_ASSERT(expr, ...) \ + (likely(expr) ? VDO_SUCCESS \ + : vdo_assertion_failed(STRINGIFY(expr), __FILE__, __LINE__, __VA_ARGS__)) + +/* Log an assertion failure message. */ +int vdo_assertion_failed(const char *expression_string, const char *file_name, + int line_number, const char *format, ...) + __printf(4, 5); + +#endif /* PERMASSERT_H */ diff --git a/drivers/md/dm-vdo/physical-zone.c b/drivers/md/dm-vdo/physical-zone.c new file mode 100644 index 000000000000..2fee3a7c1191 --- /dev/null +++ b/drivers/md/dm-vdo/physical-zone.c @@ -0,0 +1,644 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "physical-zone.h" + +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "block-map.h" +#include "completion.h" +#include "constants.h" +#include "data-vio.h" +#include "dedupe.h" +#include "encodings.h" +#include "flush.h" +#include "int-map.h" +#include "slab-depot.h" +#include "status-codes.h" +#include "vdo.h" + +/* Each user data_vio needs a PBN read lock and write lock. */ +#define LOCK_POOL_CAPACITY (2 * MAXIMUM_VDO_USER_VIOS) + +struct pbn_lock_implementation { + enum pbn_lock_type type; + const char *name; + const char *release_reason; +}; + +/* This array must have an entry for every pbn_lock_type value. */ +static const struct pbn_lock_implementation LOCK_IMPLEMENTATIONS[] = { + [VIO_READ_LOCK] = { + .type = VIO_READ_LOCK, + .name = "read", + .release_reason = "candidate duplicate", + }, + [VIO_WRITE_LOCK] = { + .type = VIO_WRITE_LOCK, + .name = "write", + .release_reason = "newly allocated", + }, + [VIO_BLOCK_MAP_WRITE_LOCK] = { + .type = VIO_BLOCK_MAP_WRITE_LOCK, + .name = "block map write", + .release_reason = "block map write", + }, +}; + +static inline bool has_lock_type(const struct pbn_lock *lock, enum pbn_lock_type type) +{ + return (lock->implementation == &LOCK_IMPLEMENTATIONS[type]); +} + +/** + * vdo_is_pbn_read_lock() - Check whether a pbn_lock is a read lock. + * @lock: The lock to check. + * + * Return: true if the lock is a read lock. + */ +bool vdo_is_pbn_read_lock(const struct pbn_lock *lock) +{ + return has_lock_type(lock, VIO_READ_LOCK); +} + +static inline void set_pbn_lock_type(struct pbn_lock *lock, enum pbn_lock_type type) +{ + lock->implementation = &LOCK_IMPLEMENTATIONS[type]; +} + +/** + * vdo_downgrade_pbn_write_lock() - Downgrade a PBN write lock to a PBN read lock. + * @lock: The PBN write lock to downgrade. + * + * The lock holder count is cleared and the caller is responsible for setting the new count. + */ +void vdo_downgrade_pbn_write_lock(struct pbn_lock *lock, bool compressed_write) +{ + VDO_ASSERT_LOG_ONLY(!vdo_is_pbn_read_lock(lock), + "PBN lock must not already have been downgraded"); + VDO_ASSERT_LOG_ONLY(!has_lock_type(lock, VIO_BLOCK_MAP_WRITE_LOCK), + "must not downgrade block map write locks"); + VDO_ASSERT_LOG_ONLY(lock->holder_count == 1, + "PBN write lock should have one holder but has %u", + lock->holder_count); + /* + * data_vio write locks are downgraded in place--the writer retains the hold on the lock. + * If this was a compressed write, the holder has not yet journaled its own inc ref, + * otherwise, it has. + */ + lock->increment_limit = + (compressed_write ? MAXIMUM_REFERENCE_COUNT : MAXIMUM_REFERENCE_COUNT - 1); + set_pbn_lock_type(lock, VIO_READ_LOCK); +} + +/** + * vdo_claim_pbn_lock_increment() - Try to claim one of the available reference count increments on + * a read lock. + * @lock: The PBN read lock from which to claim an increment. + * + * Claims may be attempted from any thread. A claim is only valid until the PBN lock is released. + * + * Return: true if the claim succeeded, guaranteeing one increment can be made without overflowing + * the PBN's reference count. + */ +bool vdo_claim_pbn_lock_increment(struct pbn_lock *lock) +{ + /* + * Claim the next free reference atomically since hash locks from multiple hash zone + * threads might be concurrently deduplicating against a single PBN lock on compressed + * block. As long as hitting the increment limit will lead to the PBN lock being released + * in a sane time-frame, we won't overflow a 32-bit claim counter, allowing a simple add + * instead of a compare-and-swap. + */ + u32 claim_number = (u32) atomic_add_return(1, &lock->increments_claimed); + + return (claim_number <= lock->increment_limit); +} + +/** + * vdo_assign_pbn_lock_provisional_reference() - Inform a PBN lock that it is responsible for a + * provisional reference. + * @lock: The PBN lock. + */ +void vdo_assign_pbn_lock_provisional_reference(struct pbn_lock *lock) +{ + VDO_ASSERT_LOG_ONLY(!lock->has_provisional_reference, + "lock does not have a provisional reference"); + lock->has_provisional_reference = true; +} + +/** + * vdo_unassign_pbn_lock_provisional_reference() - Inform a PBN lock that it is no longer + * responsible for a provisional reference. + * @lock: The PBN lock. + */ +void vdo_unassign_pbn_lock_provisional_reference(struct pbn_lock *lock) +{ + lock->has_provisional_reference = false; +} + +/** + * release_pbn_lock_provisional_reference() - If the lock is responsible for a provisional + * reference, release that reference. + * @lock: The lock. + * @locked_pbn: The PBN covered by the lock. + * @allocator: The block allocator from which to release the reference. + * + * This method is called when the lock is released. + */ +static void release_pbn_lock_provisional_reference(struct pbn_lock *lock, + physical_block_number_t locked_pbn, + struct block_allocator *allocator) +{ + int result; + + if (!vdo_pbn_lock_has_provisional_reference(lock)) + return; + + result = vdo_release_block_reference(allocator, locked_pbn); + if (result != VDO_SUCCESS) { + vdo_log_error_strerror(result, + "Failed to release reference to %s physical block %llu", + lock->implementation->release_reason, + (unsigned long long) locked_pbn); + } + + vdo_unassign_pbn_lock_provisional_reference(lock); +} + +/** + * union idle_pbn_lock - PBN lock list entries. + * + * Unused (idle) PBN locks are kept in a list. Just like in a malloc implementation, the lock + * structure is unused memory, so we can save a bit of space (and not pollute the lock structure + * proper) by using a union to overlay the lock structure with the free list. + */ +typedef union { + /** @entry: Only used while locks are in the pool. */ + struct list_head entry; + /** @lock: Only used while locks are not in the pool. */ + struct pbn_lock lock; +} idle_pbn_lock; + +/** + * struct pbn_lock_pool - list of PBN locks. + * + * The lock pool is little more than the memory allocated for the locks. + */ +struct pbn_lock_pool { + /** @capacity: The number of locks allocated for the pool. */ + size_t capacity; + /** @borrowed: The number of locks currently borrowed from the pool. */ + size_t borrowed; + /** @idle_list: A list containing all idle PBN lock instances. */ + struct list_head idle_list; + /** @locks: The memory for all the locks allocated by this pool. */ + idle_pbn_lock locks[]; +}; + +/** + * return_pbn_lock_to_pool() - Return a pbn lock to its pool. + * @pool: The pool from which the lock was borrowed. + * @lock: The last reference to the lock being returned. + * + * It must be the last live reference, as if the memory were being freed (the lock memory will + * re-initialized or zeroed). + */ +static void return_pbn_lock_to_pool(struct pbn_lock_pool *pool, struct pbn_lock *lock) +{ + idle_pbn_lock *idle; + + /* A bit expensive, but will promptly catch some use-after-free errors. */ + memset(lock, 0, sizeof(*lock)); + + idle = container_of(lock, idle_pbn_lock, lock); + INIT_LIST_HEAD(&idle->entry); + list_add_tail(&idle->entry, &pool->idle_list); + + VDO_ASSERT_LOG_ONLY(pool->borrowed > 0, "shouldn't return more than borrowed"); + pool->borrowed -= 1; +} + +/** + * make_pbn_lock_pool() - Create a new PBN lock pool and all the lock instances it can loan out. + * + * @capacity: The number of PBN locks to allocate for the pool. + * @pool_ptr: A pointer to receive the new pool. + * + * Return: VDO_SUCCESS or an error code. + */ +static int make_pbn_lock_pool(size_t capacity, struct pbn_lock_pool **pool_ptr) +{ + size_t i; + struct pbn_lock_pool *pool; + int result; + + result = vdo_allocate_extended(struct pbn_lock_pool, capacity, idle_pbn_lock, + __func__, &pool); + if (result != VDO_SUCCESS) + return result; + + pool->capacity = capacity; + pool->borrowed = capacity; + INIT_LIST_HEAD(&pool->idle_list); + + for (i = 0; i < capacity; i++) + return_pbn_lock_to_pool(pool, &pool->locks[i].lock); + + *pool_ptr = pool; + return VDO_SUCCESS; +} + +/** + * free_pbn_lock_pool() - Free a PBN lock pool. + * @pool: The lock pool to free. + * + * This also frees all the PBN locks it allocated, so the caller must ensure that all locks have + * been returned to the pool. + */ +static void free_pbn_lock_pool(struct pbn_lock_pool *pool) +{ + if (pool == NULL) + return; + + VDO_ASSERT_LOG_ONLY(pool->borrowed == 0, + "All PBN locks must be returned to the pool before it is freed, but %zu locks are still on loan", + pool->borrowed); + vdo_free(pool); +} + +/** + * borrow_pbn_lock_from_pool() - Borrow a PBN lock from the pool and initialize it with the + * provided type. + * @pool: The pool from which to borrow. + * @type: The type with which to initialize the lock. + * @lock_ptr: A pointer to receive the borrowed lock. + * + * Pools do not grow on demand or allocate memory, so this will fail if the pool is empty. Borrowed + * locks are still associated with this pool and must be returned to only this pool. + * + * Return: VDO_SUCCESS, or VDO_LOCK_ERROR if the pool is empty. + */ +static int __must_check borrow_pbn_lock_from_pool(struct pbn_lock_pool *pool, + enum pbn_lock_type type, + struct pbn_lock **lock_ptr) +{ + int result; + struct list_head *idle_entry; + idle_pbn_lock *idle; + + if (pool->borrowed >= pool->capacity) + return vdo_log_error_strerror(VDO_LOCK_ERROR, + "no free PBN locks left to borrow"); + pool->borrowed += 1; + + result = VDO_ASSERT(!list_empty(&pool->idle_list), + "idle list should not be empty if pool not at capacity"); + if (result != VDO_SUCCESS) + return result; + + idle_entry = pool->idle_list.prev; + list_del(idle_entry); + memset(idle_entry, 0, sizeof(*idle_entry)); + + idle = list_entry(idle_entry, idle_pbn_lock, entry); + idle->lock.holder_count = 0; + set_pbn_lock_type(&idle->lock, type); + + *lock_ptr = &idle->lock; + return VDO_SUCCESS; +} + +/** + * initialize_zone() - Initialize a physical zone. + * @vdo: The vdo to which the zone will belong. + * @zones: The physical_zones to which the zone being initialized belongs + * + * Return: VDO_SUCCESS or an error code. + */ +static int initialize_zone(struct vdo *vdo, struct physical_zones *zones) +{ + int result; + zone_count_t zone_number = zones->zone_count; + struct physical_zone *zone = &zones->zones[zone_number]; + + result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, &zone->pbn_operations); + if (result != VDO_SUCCESS) + return result; + + result = make_pbn_lock_pool(LOCK_POOL_CAPACITY, &zone->lock_pool); + if (result != VDO_SUCCESS) { + vdo_int_map_free(zone->pbn_operations); + return result; + } + + zone->zone_number = zone_number; + zone->thread_id = vdo->thread_config.physical_threads[zone_number]; + zone->allocator = &vdo->depot->allocators[zone_number]; + zone->next = &zones->zones[(zone_number + 1) % vdo->thread_config.physical_zone_count]; + result = vdo_make_default_thread(vdo, zone->thread_id); + if (result != VDO_SUCCESS) { + free_pbn_lock_pool(vdo_forget(zone->lock_pool)); + vdo_int_map_free(zone->pbn_operations); + return result; + } + return result; +} + +/** + * vdo_make_physical_zones() - Make the physical zones for a vdo. + * @vdo: The vdo being constructed + * @zones_ptr: A pointer to hold the zones + * + * Return: VDO_SUCCESS or an error code. + */ +int vdo_make_physical_zones(struct vdo *vdo, struct physical_zones **zones_ptr) +{ + struct physical_zones *zones; + int result; + zone_count_t zone_count = vdo->thread_config.physical_zone_count; + + if (zone_count == 0) + return VDO_SUCCESS; + + result = vdo_allocate_extended(struct physical_zones, zone_count, + struct physical_zone, __func__, &zones); + if (result != VDO_SUCCESS) + return result; + + for (zones->zone_count = 0; zones->zone_count < zone_count; zones->zone_count++) { + result = initialize_zone(vdo, zones); + if (result != VDO_SUCCESS) { + vdo_free_physical_zones(zones); + return result; + } + } + + *zones_ptr = zones; + return VDO_SUCCESS; +} + +/** + * vdo_free_physical_zones() - Destroy the physical zones. + * @zones: The zones to free. + */ +void vdo_free_physical_zones(struct physical_zones *zones) +{ + zone_count_t index; + + if (zones == NULL) + return; + + for (index = 0; index < zones->zone_count; index++) { + struct physical_zone *zone = &zones->zones[index]; + + free_pbn_lock_pool(vdo_forget(zone->lock_pool)); + vdo_int_map_free(vdo_forget(zone->pbn_operations)); + } + + vdo_free(zones); +} + +/** + * vdo_get_physical_zone_pbn_lock() - Get the lock on a PBN if one exists. + * @zone: The physical zone responsible for the PBN. + * @pbn: The physical block number whose lock is desired. + * + * Return: The lock or NULL if the PBN is not locked. + */ +struct pbn_lock *vdo_get_physical_zone_pbn_lock(struct physical_zone *zone, + physical_block_number_t pbn) +{ + return ((zone == NULL) ? NULL : vdo_int_map_get(zone->pbn_operations, pbn)); +} + +/** + * vdo_attempt_physical_zone_pbn_lock() - Attempt to lock a physical block in the zone responsible + * for it. + * @zone: The physical zone responsible for the PBN. + * @pbn: The physical block number to lock. + * @type: The type with which to initialize a new lock. + * @lock_ptr: A pointer to receive the lock, existing or new. + * + * If the PBN is already locked, the existing lock will be returned. Otherwise, a new lock instance + * will be borrowed from the pool, initialized, and returned. The lock owner will be NULL for a new + * lock acquired by the caller, who is responsible for setting that field promptly. The lock owner + * will be non-NULL when there is already an existing lock on the PBN. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_attempt_physical_zone_pbn_lock(struct physical_zone *zone, + physical_block_number_t pbn, + enum pbn_lock_type type, + struct pbn_lock **lock_ptr) +{ + /* + * Borrow and prepare a lock from the pool so we don't have to do two int_map accesses in + * the common case of no lock contention. + */ + struct pbn_lock *lock, *new_lock = NULL; + int result; + + result = borrow_pbn_lock_from_pool(zone->lock_pool, type, &new_lock); + if (result != VDO_SUCCESS) { + VDO_ASSERT_LOG_ONLY(false, "must always be able to borrow a PBN lock"); + return result; + } + + result = vdo_int_map_put(zone->pbn_operations, pbn, new_lock, false, + (void **) &lock); + if (result != VDO_SUCCESS) { + return_pbn_lock_to_pool(zone->lock_pool, new_lock); + return result; + } + + if (lock != NULL) { + /* The lock is already held, so we don't need the borrowed one. */ + return_pbn_lock_to_pool(zone->lock_pool, vdo_forget(new_lock)); + result = VDO_ASSERT(lock->holder_count > 0, "physical block %llu lock held", + (unsigned long long) pbn); + if (result != VDO_SUCCESS) + return result; + *lock_ptr = lock; + } else { + *lock_ptr = new_lock; + } + return VDO_SUCCESS; +} + +/** + * allocate_and_lock_block() - Attempt to allocate a block from this zone. + * @allocation: The struct allocation of the data_vio attempting to allocate. + * + * If a block is allocated, the recipient will also hold a lock on it. + * + * Return: VDO_SUCCESS if a block was allocated, or an error code. + */ +static int allocate_and_lock_block(struct allocation *allocation) +{ + int result; + struct pbn_lock *lock; + + VDO_ASSERT_LOG_ONLY(allocation->lock == NULL, + "must not allocate a block while already holding a lock on one"); + + result = vdo_allocate_block(allocation->zone->allocator, &allocation->pbn); + if (result != VDO_SUCCESS) + return result; + + result = vdo_attempt_physical_zone_pbn_lock(allocation->zone, allocation->pbn, + allocation->write_lock_type, &lock); + if (result != VDO_SUCCESS) + return result; + + if (lock->holder_count > 0) { + /* This block is already locked, which should be impossible. */ + return vdo_log_error_strerror(VDO_LOCK_ERROR, + "Newly allocated block %llu was spuriously locked (holder_count=%u)", + (unsigned long long) allocation->pbn, + lock->holder_count); + } + + /* We've successfully acquired a new lock, so mark it as ours. */ + lock->holder_count += 1; + allocation->lock = lock; + vdo_assign_pbn_lock_provisional_reference(lock); + return VDO_SUCCESS; +} + +/** + * retry_allocation() - Retry allocating a block now that we're done waiting for scrubbing. + * @waiter: The allocating_vio that was waiting to allocate. + * @context: The context (unused). + */ +static void retry_allocation(struct vdo_waiter *waiter, void *context __always_unused) +{ + struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); + + /* Now that some slab has scrubbed, restart the allocation process. */ + data_vio->allocation.wait_for_clean_slab = false; + data_vio->allocation.first_allocation_zone = data_vio->allocation.zone->zone_number; + continue_data_vio(data_vio); +} + +/** + * continue_allocating() - Continue searching for an allocation by enqueuing to wait for scrubbing + * or switching to the next zone. + * @data_vio: The data_vio attempting to get an allocation. + * + * This method should only be called from the error handler set in data_vio_allocate_data_block. + * + * Return: true if the allocation process has continued in another zone. + */ +static bool continue_allocating(struct data_vio *data_vio) +{ + struct allocation *allocation = &data_vio->allocation; + struct physical_zone *zone = allocation->zone; + struct vdo_completion *completion = &data_vio->vio.completion; + int result = VDO_SUCCESS; + bool was_waiting = allocation->wait_for_clean_slab; + bool tried_all = (allocation->first_allocation_zone == zone->next->zone_number); + + vdo_reset_completion(completion); + + if (tried_all && !was_waiting) { + /* + * We've already looked in all the zones, and found nothing. So go through the + * zones again, and wait for each to scrub before trying to allocate. + */ + allocation->wait_for_clean_slab = true; + allocation->first_allocation_zone = zone->zone_number; + } + + if (allocation->wait_for_clean_slab) { + data_vio->waiter.callback = retry_allocation; + result = vdo_enqueue_clean_slab_waiter(zone->allocator, + &data_vio->waiter); + if (result == VDO_SUCCESS) { + /* We've enqueued to wait for a slab to be scrubbed. */ + return true; + } + + if ((result != VDO_NO_SPACE) || (was_waiting && tried_all)) { + vdo_set_completion_result(completion, result); + return false; + } + } + + allocation->zone = zone->next; + completion->callback_thread_id = allocation->zone->thread_id; + vdo_launch_completion(completion); + return true; +} + +/** + * vdo_allocate_block_in_zone() - Attempt to allocate a block in the current physical zone, and if + * that fails try the next if possible. + * @data_vio: The data_vio needing an allocation. + * + * Return: true if a block was allocated, if not the data_vio will have been dispatched so the + * caller must not touch it. + */ +bool vdo_allocate_block_in_zone(struct data_vio *data_vio) +{ + int result = allocate_and_lock_block(&data_vio->allocation); + + if (result == VDO_SUCCESS) + return true; + + if ((result != VDO_NO_SPACE) || !continue_allocating(data_vio)) + continue_data_vio_with_error(data_vio, result); + + return false; +} + +/** + * vdo_release_physical_zone_pbn_lock() - Release a physical block lock if it is held and return it + * to the lock pool. + * @zone: The physical zone in which the lock was obtained. + * @locked_pbn: The physical block number to unlock. + * @lock: The lock being released. + * + * It must be the last live reference, as if the memory were being freed (the + * lock memory will re-initialized or zeroed). + */ +void vdo_release_physical_zone_pbn_lock(struct physical_zone *zone, + physical_block_number_t locked_pbn, + struct pbn_lock *lock) +{ + struct pbn_lock *holder; + + if (lock == NULL) + return; + + VDO_ASSERT_LOG_ONLY(lock->holder_count > 0, + "should not be releasing a lock that is not held"); + + lock->holder_count -= 1; + if (lock->holder_count > 0) { + /* The lock was shared and is still referenced, so don't release it yet. */ + return; + } + + holder = vdo_int_map_remove(zone->pbn_operations, locked_pbn); + VDO_ASSERT_LOG_ONLY((lock == holder), "physical block lock mismatch for block %llu", + (unsigned long long) locked_pbn); + + release_pbn_lock_provisional_reference(lock, locked_pbn, zone->allocator); + return_pbn_lock_to_pool(zone->lock_pool, lock); +} + +/** + * vdo_dump_physical_zone() - Dump information about a physical zone to the log for debugging. + * @zone: The zone to dump. + */ +void vdo_dump_physical_zone(const struct physical_zone *zone) +{ + vdo_dump_block_allocator(zone->allocator); +} diff --git a/drivers/md/dm-vdo/physical-zone.h b/drivers/md/dm-vdo/physical-zone.h new file mode 100644 index 000000000000..47d874fd5a0b --- /dev/null +++ b/drivers/md/dm-vdo/physical-zone.h @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_PHYSICAL_ZONE_H +#define VDO_PHYSICAL_ZONE_H + +#include + +#include "types.h" + +/* + * The type of a PBN lock. + */ +enum pbn_lock_type { + VIO_READ_LOCK, + VIO_WRITE_LOCK, + VIO_BLOCK_MAP_WRITE_LOCK, +}; + +struct pbn_lock_implementation; + +/* + * A PBN lock. + */ +struct pbn_lock { + /* The implementation of the lock */ + const struct pbn_lock_implementation *implementation; + + /* The number of VIOs holding or sharing this lock */ + data_vio_count_t holder_count; + /* + * The number of compressed block writers holding a share of this lock while they are + * acquiring a reference to the PBN. + */ + u8 fragment_locks; + + /* Whether the locked PBN has been provisionally referenced on behalf of the lock holder. */ + bool has_provisional_reference; + + /* + * For read locks, the number of references that were known to be available on the locked + * block at the time the lock was acquired. + */ + u8 increment_limit; + + /* + * For read locks, the number of data_vios that have tried to claim one of the available + * increments during the lifetime of the lock. Each claim will first increment this + * counter, so it can exceed the increment limit. + */ + atomic_t increments_claimed; +}; + +struct physical_zone { + /* Which physical zone this is */ + zone_count_t zone_number; + /* The thread ID for this zone */ + thread_id_t thread_id; + /* In progress operations keyed by PBN */ + struct int_map *pbn_operations; + /* Pool of unused pbn_lock instances */ + struct pbn_lock_pool *lock_pool; + /* The block allocator for this zone */ + struct block_allocator *allocator; + /* The next zone from which to attempt an allocation */ + struct physical_zone *next; +}; + +struct physical_zones { + /* The number of zones */ + zone_count_t zone_count; + /* The physical zones themselves */ + struct physical_zone zones[]; +}; + +bool __must_check vdo_is_pbn_read_lock(const struct pbn_lock *lock); +void vdo_downgrade_pbn_write_lock(struct pbn_lock *lock, bool compressed_write); +bool __must_check vdo_claim_pbn_lock_increment(struct pbn_lock *lock); + +/** + * vdo_pbn_lock_has_provisional_reference() - Check whether a PBN lock has a provisional reference. + * @lock: The PBN lock. + */ +static inline bool vdo_pbn_lock_has_provisional_reference(struct pbn_lock *lock) +{ + return ((lock != NULL) && lock->has_provisional_reference); +} + +void vdo_assign_pbn_lock_provisional_reference(struct pbn_lock *lock); +void vdo_unassign_pbn_lock_provisional_reference(struct pbn_lock *lock); + +int __must_check vdo_make_physical_zones(struct vdo *vdo, + struct physical_zones **zones_ptr); + +void vdo_free_physical_zones(struct physical_zones *zones); + +struct pbn_lock * __must_check vdo_get_physical_zone_pbn_lock(struct physical_zone *zone, + physical_block_number_t pbn); + +int __must_check vdo_attempt_physical_zone_pbn_lock(struct physical_zone *zone, + physical_block_number_t pbn, + enum pbn_lock_type type, + struct pbn_lock **lock_ptr); + +bool __must_check vdo_allocate_block_in_zone(struct data_vio *data_vio); + +void vdo_release_physical_zone_pbn_lock(struct physical_zone *zone, + physical_block_number_t locked_pbn, + struct pbn_lock *lock); + +void vdo_dump_physical_zone(const struct physical_zone *zone); + +#endif /* VDO_PHYSICAL_ZONE_H */ diff --git a/drivers/md/dm-vdo/priority-table.c b/drivers/md/dm-vdo/priority-table.c new file mode 100644 index 000000000000..42d3d8d0e4b5 --- /dev/null +++ b/drivers/md/dm-vdo/priority-table.c @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "priority-table.h" + +#include + +#include "errors.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "status-codes.h" + +/* We use a single 64-bit search vector, so the maximum priority is 63 */ +#define MAX_PRIORITY 63 + +/* + * All the entries with the same priority are queued in a circular list in a bucket for that + * priority. The table is essentially an array of buckets. + */ +struct bucket { + /* + * The head of a queue of table entries, all having the same priority + */ + struct list_head queue; + /* The priority of all the entries in this bucket */ + unsigned int priority; +}; + +/* + * A priority table is an array of buckets, indexed by priority. New entries are added to the end + * of the queue in the appropriate bucket. The dequeue operation finds the highest-priority + * non-empty bucket by searching a bit vector represented as a single 8-byte word, which is very + * fast with compiler and CPU support. + */ +struct priority_table { + /* The maximum priority of entries that may be stored in this table */ + unsigned int max_priority; + /* A bit vector flagging all buckets that are currently non-empty */ + u64 search_vector; + /* The array of all buckets, indexed by priority */ + struct bucket buckets[]; +}; + +/** + * vdo_make_priority_table() - Allocate and initialize a new priority_table. + * @max_priority: The maximum priority value for table entries. + * @table_ptr: A pointer to hold the new table. + * + * Return: VDO_SUCCESS or an error code. + */ +int vdo_make_priority_table(unsigned int max_priority, struct priority_table **table_ptr) +{ + struct priority_table *table; + int result; + unsigned int priority; + + if (max_priority > MAX_PRIORITY) + return UDS_INVALID_ARGUMENT; + + result = vdo_allocate_extended(struct priority_table, max_priority + 1, + struct bucket, __func__, &table); + if (result != VDO_SUCCESS) + return result; + + for (priority = 0; priority <= max_priority; priority++) { + struct bucket *bucket = &table->buckets[priority]; + + bucket->priority = priority; + INIT_LIST_HEAD(&bucket->queue); + } + + table->max_priority = max_priority; + table->search_vector = 0; + + *table_ptr = table; + return VDO_SUCCESS; +} + +/** + * vdo_free_priority_table() - Free a priority_table. + * @table: The table to free. + * + * The table does not own the entries stored in it and they are not freed by this call. + */ +void vdo_free_priority_table(struct priority_table *table) +{ + if (table == NULL) + return; + + /* + * Unlink the buckets from any entries still in the table so the entries won't be left with + * dangling pointers to freed memory. + */ + vdo_reset_priority_table(table); + + vdo_free(table); +} + +/** + * vdo_reset_priority_table() - Reset a priority table, leaving it in the same empty state as when + * newly constructed. + * @table: The table to reset. + * + * The table does not own the entries stored in it and they are not freed (or even unlinked from + * each other) by this call. + */ +void vdo_reset_priority_table(struct priority_table *table) +{ + unsigned int priority; + + table->search_vector = 0; + for (priority = 0; priority <= table->max_priority; priority++) + list_del_init(&table->buckets[priority].queue); +} + +/** + * vdo_priority_table_enqueue() - Add a new entry to the priority table, appending it to the queue + * for entries with the specified priority. + * @table: The table in which to store the entry. + * @priority: The priority of the entry. + * @entry: The list_head embedded in the entry to store in the table (the caller must have + * initialized it). + */ +void vdo_priority_table_enqueue(struct priority_table *table, unsigned int priority, + struct list_head *entry) +{ + VDO_ASSERT_LOG_ONLY((priority <= table->max_priority), + "entry priority must be valid for the table"); + + /* Append the entry to the queue in the specified bucket. */ + list_move_tail(entry, &table->buckets[priority].queue); + + /* Flag the bucket in the search vector since it must be non-empty. */ + table->search_vector |= (1ULL << priority); +} + +static inline void mark_bucket_empty(struct priority_table *table, struct bucket *bucket) +{ + table->search_vector &= ~(1ULL << bucket->priority); +} + +/** + * vdo_priority_table_dequeue() - Find the highest-priority entry in the table, remove it from the + * table, and return it. + * @table: The priority table from which to remove an entry. + * + * If there are multiple entries with the same priority, the one that has been in the table with + * that priority the longest will be returned. + * + * Return: The dequeued entry, or NULL if the table is currently empty. + */ +struct list_head *vdo_priority_table_dequeue(struct priority_table *table) +{ + struct bucket *bucket; + struct list_head *entry; + int top_priority; + + if (table->search_vector == 0) { + /* All buckets are empty. */ + return NULL; + } + + /* + * Find the highest priority non-empty bucket by finding the highest-order non-zero bit in + * the search vector. + */ + top_priority = ilog2(table->search_vector); + + /* Dequeue the first entry in the bucket. */ + bucket = &table->buckets[top_priority]; + entry = bucket->queue.next; + list_del_init(entry); + + /* Clear the bit in the search vector if the bucket has been emptied. */ + if (list_empty(&bucket->queue)) + mark_bucket_empty(table, bucket); + + return entry; +} + +/** + * vdo_priority_table_remove() - Remove a specified entry from its priority table. + * @table: The table from which to remove the entry. + * @entry: The entry to remove from the table. + */ +void vdo_priority_table_remove(struct priority_table *table, struct list_head *entry) +{ + struct list_head *next_entry; + + /* + * We can't guard against calls where the entry is on a list for a different table, but + * it's easy to deal with an entry not in any table or list. + */ + if (list_empty(entry)) + return; + + /* + * Remove the entry from the bucket list, remembering a pointer to another entry in the + * ring. + */ + next_entry = entry->next; + list_del_init(entry); + + /* + * If the rest of the list is now empty, the next node must be the list head in the bucket + * and we can use it to update the search vector. + */ + if (list_empty(next_entry)) + mark_bucket_empty(table, list_entry(next_entry, struct bucket, queue)); +} + +/** + * vdo_is_priority_table_empty() - Return whether the priority table is empty. + * @table: The table to check. + * + * Return: true if the table is empty. + */ +bool vdo_is_priority_table_empty(struct priority_table *table) +{ + return (table->search_vector == 0); +} diff --git a/drivers/md/dm-vdo/priority-table.h b/drivers/md/dm-vdo/priority-table.h new file mode 100644 index 000000000000..8b060462e3e4 --- /dev/null +++ b/drivers/md/dm-vdo/priority-table.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_PRIORITY_TABLE_H +#define VDO_PRIORITY_TABLE_H + +#include + +/* + * A priority_table is a simple implementation of a priority queue for entries with priorities that + * are small non-negative integer values. It implements the obvious priority queue operations of + * enqueuing an entry and dequeuing an entry with the maximum priority. It also supports removing + * an arbitrary entry. The priority of an entry already in the table can be changed by removing it + * and re-enqueuing it with a different priority. All operations have O(1) complexity. + * + * The links for the table entries must be embedded in the entries themselves. Lists are used to + * link entries in the table and no wrapper type is declared, so an existing list entry in an + * object can also be used to queue it in a priority_table, assuming the field is not used for + * anything else while so queued. + * + * The table is implemented as an array of queues (circular lists) indexed by priority, along with + * a hint for which queues are non-empty. Steven Skiena calls a very similar structure a "bounded + * height priority queue", but given the resemblance to a hash table, "priority table" seems both + * shorter and more apt, if somewhat novel. + */ + +struct priority_table; + +int __must_check vdo_make_priority_table(unsigned int max_priority, + struct priority_table **table_ptr); + +void vdo_free_priority_table(struct priority_table *table); + +void vdo_priority_table_enqueue(struct priority_table *table, unsigned int priority, + struct list_head *entry); + +void vdo_reset_priority_table(struct priority_table *table); + +struct list_head * __must_check vdo_priority_table_dequeue(struct priority_table *table); + +void vdo_priority_table_remove(struct priority_table *table, struct list_head *entry); + +bool __must_check vdo_is_priority_table_empty(struct priority_table *table); + +#endif /* VDO_PRIORITY_TABLE_H */ diff --git a/drivers/md/dm-vdo/recovery-journal.c b/drivers/md/dm-vdo/recovery-journal.c new file mode 100644 index 000000000000..ee6321a3e523 --- /dev/null +++ b/drivers/md/dm-vdo/recovery-journal.c @@ -0,0 +1,1762 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "recovery-journal.h" + +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "block-map.h" +#include "completion.h" +#include "constants.h" +#include "data-vio.h" +#include "encodings.h" +#include "io-submitter.h" +#include "slab-depot.h" +#include "types.h" +#include "vdo.h" +#include "vio.h" +#include "wait-queue.h" + +static const u64 RECOVERY_COUNT_MASK = 0xff; + +/* + * The number of reserved blocks must be large enough to prevent a new recovery journal + * block write from overwriting a block which appears to still be a valid head block of the + * journal. Currently, that means reserving enough space for all 2048 data_vios. + */ +#define RECOVERY_JOURNAL_RESERVED_BLOCKS \ + ((MAXIMUM_VDO_USER_VIOS / RECOVERY_JOURNAL_ENTRIES_PER_BLOCK) + 2) + +/** + * DOC: Lock Counters. + * + * A lock_counter is intended to keep all of the locks for the blocks in the recovery journal. The + * per-zone counters are all kept in a single array which is arranged by zone (i.e. zone 0's lock 0 + * is at index 0, zone 0's lock 1 is at index 1, and zone 1's lock 0 is at index 'locks'. This + * arrangement is intended to minimize cache-line contention for counters from different zones. + * + * The locks are implemented as a single object instead of as a lock counter per lock both to + * afford this opportunity to reduce cache line contention and also to eliminate the need to have a + * completion per lock. + * + * Lock sets are laid out with the set for recovery journal first, followed by the logical zones, + * and then the physical zones. + */ + +enum lock_counter_state { + LOCK_COUNTER_STATE_NOT_NOTIFYING, + LOCK_COUNTER_STATE_NOTIFYING, + LOCK_COUNTER_STATE_SUSPENDED, +}; + +/** + * get_zone_count_ptr() - Get a pointer to the zone count for a given lock on a given zone. + * @journal: The recovery journal. + * @lock_number: The lock to get. + * @zone_type: The zone type whose count is desired. + * + * Return: A pointer to the zone count for the given lock and zone. + */ +static inline atomic_t *get_zone_count_ptr(struct recovery_journal *journal, + block_count_t lock_number, + enum vdo_zone_type zone_type) +{ + return ((zone_type == VDO_ZONE_TYPE_LOGICAL) + ? &journal->lock_counter.logical_zone_counts[lock_number] + : &journal->lock_counter.physical_zone_counts[lock_number]); +} + +/** + * get_counter() - Get the zone counter for a given lock on a given zone. + * @journal: The recovery journal. + * @lock_number: The lock to get. + * @zone_type: The zone type whose count is desired. + * @zone_id: The zone index whose count is desired. + * + * Return: The counter for the given lock and zone. + */ +static inline u16 *get_counter(struct recovery_journal *journal, + block_count_t lock_number, enum vdo_zone_type zone_type, + zone_count_t zone_id) +{ + struct lock_counter *counter = &journal->lock_counter; + block_count_t zone_counter = (counter->locks * zone_id) + lock_number; + + if (zone_type == VDO_ZONE_TYPE_JOURNAL) + return &counter->journal_counters[zone_counter]; + + if (zone_type == VDO_ZONE_TYPE_LOGICAL) + return &counter->logical_counters[zone_counter]; + + return &counter->physical_counters[zone_counter]; +} + +static atomic_t *get_decrement_counter(struct recovery_journal *journal, + block_count_t lock_number) +{ + return &journal->lock_counter.journal_decrement_counts[lock_number]; +} + +/** + * is_journal_zone_locked() - Check whether the journal zone is locked for a given lock. + * @journal: The recovery journal. + * @lock_number: The lock to check. + * + * Return: true if the journal zone is locked. + */ +static bool is_journal_zone_locked(struct recovery_journal *journal, + block_count_t lock_number) +{ + u16 journal_value = *get_counter(journal, lock_number, VDO_ZONE_TYPE_JOURNAL, 0); + u32 decrements = atomic_read(get_decrement_counter(journal, lock_number)); + + /* Pairs with barrier in vdo_release_journal_entry_lock() */ + smp_rmb(); + VDO_ASSERT_LOG_ONLY((decrements <= journal_value), + "journal zone lock counter must not underflow"); + return (journal_value != decrements); +} + +/** + * vdo_release_recovery_journal_block_reference() - Release a reference to a recovery journal + * block. + * @journal: The recovery journal. + * @sequence_number: The journal sequence number of the referenced block. + * @zone_type: The type of the zone making the adjustment. + * @zone_id: The ID of the zone making the adjustment. + * + * If this is the last reference for a given zone type, an attempt will be made to reap the + * journal. + */ +void vdo_release_recovery_journal_block_reference(struct recovery_journal *journal, + sequence_number_t sequence_number, + enum vdo_zone_type zone_type, + zone_count_t zone_id) +{ + u16 *current_value; + block_count_t lock_number; + int prior_state; + + if (sequence_number == 0) + return; + + lock_number = vdo_get_recovery_journal_block_number(journal, sequence_number); + current_value = get_counter(journal, lock_number, zone_type, zone_id); + + VDO_ASSERT_LOG_ONLY((*current_value >= 1), + "decrement of lock counter must not underflow"); + *current_value -= 1; + + if (zone_type == VDO_ZONE_TYPE_JOURNAL) { + if (is_journal_zone_locked(journal, lock_number)) + return; + } else { + atomic_t *zone_count; + + if (*current_value != 0) + return; + + zone_count = get_zone_count_ptr(journal, lock_number, zone_type); + + if (atomic_add_return(-1, zone_count) > 0) + return; + } + + /* + * Extra barriers because this was original developed using a CAS operation that implicitly + * had them. + */ + smp_mb__before_atomic(); + prior_state = atomic_cmpxchg(&journal->lock_counter.state, + LOCK_COUNTER_STATE_NOT_NOTIFYING, + LOCK_COUNTER_STATE_NOTIFYING); + /* same as before_atomic */ + smp_mb__after_atomic(); + + if (prior_state != LOCK_COUNTER_STATE_NOT_NOTIFYING) + return; + + vdo_launch_completion(&journal->lock_counter.completion); +} + +static inline struct recovery_journal_block * __must_check get_journal_block(struct list_head *list) +{ + return list_first_entry_or_null(list, struct recovery_journal_block, list_node); +} + +/** + * pop_free_list() - Get a block from the end of the free list. + * @journal: The journal. + * + * Return: The block or NULL if the list is empty. + */ +static struct recovery_journal_block * __must_check pop_free_list(struct recovery_journal *journal) +{ + struct recovery_journal_block *block; + + if (list_empty(&journal->free_tail_blocks)) + return NULL; + + block = list_last_entry(&journal->free_tail_blocks, + struct recovery_journal_block, list_node); + list_del_init(&block->list_node); + return block; +} + +/** + * is_block_dirty() - Check whether a recovery block is dirty. + * @block: The block to check. + * + * Indicates it has any uncommitted entries, which includes both entries not written and entries + * written but not yet acknowledged. + * + * Return: true if the block has any uncommitted entries. + */ +static inline bool __must_check is_block_dirty(const struct recovery_journal_block *block) +{ + return (block->uncommitted_entry_count > 0); +} + +/** + * is_block_empty() - Check whether a journal block is empty. + * @block: The block to check. + * + * Return: true if the block has no entries. + */ +static inline bool __must_check is_block_empty(const struct recovery_journal_block *block) +{ + return (block->entry_count == 0); +} + +/** + * is_block_full() - Check whether a journal block is full. + * @block: The block to check. + * + * Return: true if the block is full. + */ +static inline bool __must_check is_block_full(const struct recovery_journal_block *block) +{ + return ((block == NULL) || (block->journal->entries_per_block == block->entry_count)); +} + +/** + * assert_on_journal_thread() - Assert that we are running on the journal thread. + * @journal: The journal. + * @function_name: The function doing the check (for logging). + */ +static void assert_on_journal_thread(struct recovery_journal *journal, + const char *function_name) +{ + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == journal->thread_id), + "%s() called on journal thread", function_name); +} + +/** + * continue_waiter() - Release a data_vio from the journal. + * + * Invoked whenever a data_vio is to be released from the journal, either because its entry was + * committed to disk, or because there was an error. Implements waiter_callback_fn. + */ +static void continue_waiter(struct vdo_waiter *waiter, void *context) +{ + continue_data_vio_with_error(vdo_waiter_as_data_vio(waiter), *((int *) context)); +} + +/** + * has_block_waiters() - Check whether the journal has any waiters on any blocks. + * @journal: The journal in question. + * + * Return: true if any block has a waiter. + */ +static inline bool has_block_waiters(struct recovery_journal *journal) +{ + struct recovery_journal_block *block = get_journal_block(&journal->active_tail_blocks); + + /* + * Either the first active tail block (if it exists) has waiters, or no active tail block + * has waiters. + */ + return ((block != NULL) && + (vdo_waitq_has_waiters(&block->entry_waiters) || + vdo_waitq_has_waiters(&block->commit_waiters))); +} + +static void recycle_journal_blocks(struct recovery_journal *journal); +static void recycle_journal_block(struct recovery_journal_block *block); +static void notify_commit_waiters(struct recovery_journal *journal); + +/** + * suspend_lock_counter() - Prevent the lock counter from notifying. + * @counter: The counter. + * + * Return: true if the lock counter was not notifying and hence the suspend was efficacious. + */ +static bool suspend_lock_counter(struct lock_counter *counter) +{ + int prior_state; + + /* + * Extra barriers because this was originally developed using a CAS operation that + * implicitly had them. + */ + smp_mb__before_atomic(); + prior_state = atomic_cmpxchg(&counter->state, LOCK_COUNTER_STATE_NOT_NOTIFYING, + LOCK_COUNTER_STATE_SUSPENDED); + /* same as before_atomic */ + smp_mb__after_atomic(); + + return ((prior_state == LOCK_COUNTER_STATE_SUSPENDED) || + (prior_state == LOCK_COUNTER_STATE_NOT_NOTIFYING)); +} + +static inline bool is_read_only(struct recovery_journal *journal) +{ + return vdo_is_read_only(journal->flush_vio->completion.vdo); +} + +/** + * check_for_drain_complete() - Check whether the journal has drained. + * @journal: The journal which may have just drained. + */ +static void check_for_drain_complete(struct recovery_journal *journal) +{ + int result = VDO_SUCCESS; + + if (is_read_only(journal)) { + result = VDO_READ_ONLY; + /* + * Clean up any full active blocks which were not written due to read-only mode. + * + * FIXME: This would probably be better as a short-circuit in write_block(). + */ + notify_commit_waiters(journal); + recycle_journal_blocks(journal); + + /* Release any data_vios waiting to be assigned entries. */ + vdo_waitq_notify_all_waiters(&journal->entry_waiters, + continue_waiter, &result); + } + + if (!vdo_is_state_draining(&journal->state) || + journal->reaping || + has_block_waiters(journal) || + vdo_waitq_has_waiters(&journal->entry_waiters) || + !suspend_lock_counter(&journal->lock_counter)) + return; + + if (vdo_is_state_saving(&journal->state)) { + if (journal->active_block != NULL) { + VDO_ASSERT_LOG_ONLY(((result == VDO_READ_ONLY) || + !is_block_dirty(journal->active_block)), + "journal being saved has clean active block"); + recycle_journal_block(journal->active_block); + } + + VDO_ASSERT_LOG_ONLY(list_empty(&journal->active_tail_blocks), + "all blocks in a journal being saved must be inactive"); + } + + vdo_finish_draining_with_result(&journal->state, result); +} + +/** + * notify_recovery_journal_of_read_only_mode() - Notify a recovery journal that the VDO has gone + * read-only. + * @listener: The journal. + * @parent: The completion to notify in order to acknowledge the notification. + * + * Implements vdo_read_only_notification_fn. + */ +static void notify_recovery_journal_of_read_only_mode(void *listener, + struct vdo_completion *parent) +{ + check_for_drain_complete(listener); + vdo_finish_completion(parent); +} + +/** + * enter_journal_read_only_mode() - Put the journal in read-only mode. + * @journal: The journal which has failed. + * @error_code: The error result triggering this call. + * + * All attempts to add entries after this function is called will fail. All VIOs waiting for + * commits will be awakened with an error. + */ +static void enter_journal_read_only_mode(struct recovery_journal *journal, + int error_code) +{ + vdo_enter_read_only_mode(journal->flush_vio->completion.vdo, error_code); + check_for_drain_complete(journal); +} + +/** + * vdo_get_recovery_journal_current_sequence_number() - Obtain the recovery journal's current + * sequence number. + * @journal: The journal in question. + * + * Exposed only so the block map can be initialized therefrom. + * + * Return: The sequence number of the tail block. + */ +sequence_number_t vdo_get_recovery_journal_current_sequence_number(struct recovery_journal *journal) +{ + return journal->tail; +} + +/** + * get_recovery_journal_head() - Get the head of the recovery journal. + * @journal: The journal. + * + * The head is the lowest sequence number of the block map head and the slab journal head. + * + * Return: the head of the journal. + */ +static inline sequence_number_t get_recovery_journal_head(const struct recovery_journal *journal) +{ + return min(journal->block_map_head, journal->slab_journal_head); +} + +/** + * compute_recovery_count_byte() - Compute the recovery count byte for a given recovery count. + * @recovery_count: The recovery count. + * + * Return: The byte corresponding to the recovery count. + */ +static inline u8 __must_check compute_recovery_count_byte(u64 recovery_count) +{ + return (u8)(recovery_count & RECOVERY_COUNT_MASK); +} + +/** + * check_slab_journal_commit_threshold() - Check whether the journal is over the threshold, and if + * so, force the oldest slab journal tail block to commit. + * @journal: The journal. + */ +static void check_slab_journal_commit_threshold(struct recovery_journal *journal) +{ + block_count_t current_length = journal->tail - journal->slab_journal_head; + + if (current_length > journal->slab_journal_commit_threshold) { + journal->events.slab_journal_commits_requested++; + vdo_commit_oldest_slab_journal_tail_blocks(journal->depot, + journal->slab_journal_head); + } +} + +static void reap_recovery_journal(struct recovery_journal *journal); +static void assign_entries(struct recovery_journal *journal); + +/** + * finish_reaping() - Finish reaping the journal. + * @journal: The journal being reaped. + */ +static void finish_reaping(struct recovery_journal *journal) +{ + block_count_t blocks_reaped; + sequence_number_t old_head = get_recovery_journal_head(journal); + + journal->block_map_head = journal->block_map_reap_head; + journal->slab_journal_head = journal->slab_journal_reap_head; + blocks_reaped = get_recovery_journal_head(journal) - old_head; + journal->available_space += blocks_reaped * journal->entries_per_block; + journal->reaping = false; + check_slab_journal_commit_threshold(journal); + assign_entries(journal); + check_for_drain_complete(journal); +} + +/** + * complete_reaping() - Finish reaping the journal after flushing the lower layer. + * @completion: The journal's flush VIO. + * + * This is the callback registered in reap_recovery_journal(). + */ +static void complete_reaping(struct vdo_completion *completion) +{ + struct recovery_journal *journal = completion->parent; + + finish_reaping(journal); + + /* Try reaping again in case more locks were released while flush was out. */ + reap_recovery_journal(journal); +} + +/** + * handle_flush_error() - Handle an error when flushing the lower layer due to reaping. + * @completion: The journal's flush VIO. + */ +static void handle_flush_error(struct vdo_completion *completion) +{ + struct recovery_journal *journal = completion->parent; + + vio_record_metadata_io_error(as_vio(completion)); + journal->reaping = false; + enter_journal_read_only_mode(journal, completion->result); +} + +static void flush_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct recovery_journal *journal = vio->completion.parent; + + continue_vio_after_io(vio, complete_reaping, journal->thread_id); +} + +/** + * initialize_journal_state() - Set all journal fields appropriately to start journaling from the + * current active block. + * @journal: The journal to be reset based on its active block. + */ +static void initialize_journal_state(struct recovery_journal *journal) +{ + journal->append_point.sequence_number = journal->tail; + journal->last_write_acknowledged = journal->tail; + journal->block_map_head = journal->tail; + journal->slab_journal_head = journal->tail; + journal->block_map_reap_head = journal->tail; + journal->slab_journal_reap_head = journal->tail; + journal->block_map_head_block_number = + vdo_get_recovery_journal_block_number(journal, journal->block_map_head); + journal->slab_journal_head_block_number = + vdo_get_recovery_journal_block_number(journal, + journal->slab_journal_head); + journal->available_space = + (journal->entries_per_block * vdo_get_recovery_journal_length(journal->size)); +} + +/** + * vdo_get_recovery_journal_length() - Get the number of usable recovery journal blocks. + * @journal_size: The size of the recovery journal in blocks. + * + * Return: the number of recovery journal blocks usable for entries. + */ +block_count_t vdo_get_recovery_journal_length(block_count_t journal_size) +{ + block_count_t reserved_blocks = journal_size / 4; + + if (reserved_blocks > RECOVERY_JOURNAL_RESERVED_BLOCKS) + reserved_blocks = RECOVERY_JOURNAL_RESERVED_BLOCKS; + return (journal_size - reserved_blocks); +} + +/** + * reap_recovery_journal_callback() - Attempt to reap the journal. + * @completion: The lock counter completion. + * + * Attempts to reap the journal now that all the locks on some journal block have been released. + * This is the callback registered with the lock counter. + */ +static void reap_recovery_journal_callback(struct vdo_completion *completion) +{ + struct recovery_journal *journal = (struct recovery_journal *) completion->parent; + /* + * The acknowledgment must be done before reaping so that there is no race between + * acknowledging the notification and unlocks wishing to notify. + */ + smp_wmb(); + atomic_set(&journal->lock_counter.state, LOCK_COUNTER_STATE_NOT_NOTIFYING); + + if (vdo_is_state_quiescing(&journal->state)) { + /* + * Don't start reaping when the journal is trying to quiesce. Do check if this + * notification is the last thing the is waiting on. + */ + check_for_drain_complete(journal); + return; + } + + reap_recovery_journal(journal); + check_slab_journal_commit_threshold(journal); +} + +/** + * initialize_lock_counter() - Initialize a lock counter. + * + * @journal: The recovery journal. + * @vdo: The vdo. + * + * Return: VDO_SUCCESS or an error. + */ +static int __must_check initialize_lock_counter(struct recovery_journal *journal, + struct vdo *vdo) +{ + int result; + struct thread_config *config = &vdo->thread_config; + struct lock_counter *counter = &journal->lock_counter; + + result = vdo_allocate(journal->size, u16, __func__, &counter->journal_counters); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate(journal->size, atomic_t, __func__, + &counter->journal_decrement_counts); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate(journal->size * config->logical_zone_count, u16, __func__, + &counter->logical_counters); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate(journal->size, atomic_t, __func__, + &counter->logical_zone_counts); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate(journal->size * config->physical_zone_count, u16, __func__, + &counter->physical_counters); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate(journal->size, atomic_t, __func__, + &counter->physical_zone_counts); + if (result != VDO_SUCCESS) + return result; + + vdo_initialize_completion(&counter->completion, vdo, + VDO_LOCK_COUNTER_COMPLETION); + vdo_prepare_completion(&counter->completion, reap_recovery_journal_callback, + reap_recovery_journal_callback, config->journal_thread, + journal); + counter->logical_zones = config->logical_zone_count; + counter->physical_zones = config->physical_zone_count; + counter->locks = journal->size; + return VDO_SUCCESS; +} + +/** + * set_journal_tail() - Set the journal's tail sequence number. + * @journal: The journal whose tail is to be set. + * @tail: The new tail value. + */ +static void set_journal_tail(struct recovery_journal *journal, sequence_number_t tail) +{ + /* VDO does not support sequence numbers above 1 << 48 in the slab journal. */ + if (tail >= (1ULL << 48)) + enter_journal_read_only_mode(journal, VDO_JOURNAL_OVERFLOW); + + journal->tail = tail; +} + +/** + * initialize_recovery_block() - Initialize a journal block. + * @vdo: The vdo from which to construct vios. + * @journal: The journal to which the block will belong. + * @block: The block to initialize. + * + * Return: VDO_SUCCESS or an error. + */ +static int initialize_recovery_block(struct vdo *vdo, struct recovery_journal *journal, + struct recovery_journal_block *block) +{ + char *data; + int result; + + /* + * Ensure that a block is large enough to store RECOVERY_JOURNAL_ENTRIES_PER_BLOCK entries. + */ + BUILD_BUG_ON(RECOVERY_JOURNAL_ENTRIES_PER_BLOCK > + ((VDO_BLOCK_SIZE - sizeof(struct packed_journal_header)) / + sizeof(struct packed_recovery_journal_entry))); + + /* + * Allocate a full block for the journal block even though not all of the space is used + * since the VIO needs to write a full disk block. + */ + result = vdo_allocate(VDO_BLOCK_SIZE, char, __func__, &data); + if (result != VDO_SUCCESS) + return result; + + result = allocate_vio_components(vdo, VIO_TYPE_RECOVERY_JOURNAL, + VIO_PRIORITY_HIGH, block, 1, data, &block->vio); + if (result != VDO_SUCCESS) { + vdo_free(data); + return result; + } + + list_add_tail(&block->list_node, &journal->free_tail_blocks); + block->journal = journal; + return VDO_SUCCESS; +} + +/** + * vdo_decode_recovery_journal() - Make a recovery journal and initialize it with the state that + * was decoded from the super block. + * + * @state: The decoded state of the journal. + * @nonce: The nonce of the VDO. + * @vdo: The VDO. + * @partition: The partition for the journal. + * @recovery_count: The VDO's number of completed recoveries. + * @journal_size: The number of blocks in the journal on disk. + * @journal_ptr: The pointer to hold the new recovery journal. + * + * Return: A success or error code. + */ +int vdo_decode_recovery_journal(struct recovery_journal_state_7_0 state, nonce_t nonce, + struct vdo *vdo, struct partition *partition, + u64 recovery_count, block_count_t journal_size, + struct recovery_journal **journal_ptr) +{ + block_count_t i; + struct recovery_journal *journal; + int result; + + result = vdo_allocate_extended(struct recovery_journal, + RECOVERY_JOURNAL_RESERVED_BLOCKS, + struct recovery_journal_block, __func__, + &journal); + if (result != VDO_SUCCESS) + return result; + + INIT_LIST_HEAD(&journal->free_tail_blocks); + INIT_LIST_HEAD(&journal->active_tail_blocks); + vdo_waitq_init(&journal->pending_writes); + + journal->thread_id = vdo->thread_config.journal_thread; + journal->origin = partition->offset; + journal->nonce = nonce; + journal->recovery_count = compute_recovery_count_byte(recovery_count); + journal->size = journal_size; + journal->slab_journal_commit_threshold = (journal_size * 2) / 3; + journal->logical_blocks_used = state.logical_blocks_used; + journal->block_map_data_blocks = state.block_map_data_blocks; + journal->entries_per_block = RECOVERY_JOURNAL_ENTRIES_PER_BLOCK; + set_journal_tail(journal, state.journal_start); + initialize_journal_state(journal); + /* TODO: this will have to change if we make initial resume of a VDO a real resume */ + vdo_set_admin_state_code(&journal->state, VDO_ADMIN_STATE_SUSPENDED); + + for (i = 0; i < RECOVERY_JOURNAL_RESERVED_BLOCKS; i++) { + struct recovery_journal_block *block = &journal->blocks[i]; + + result = initialize_recovery_block(vdo, journal, block); + if (result != VDO_SUCCESS) { + vdo_free_recovery_journal(journal); + return result; + } + } + + result = initialize_lock_counter(journal, vdo); + if (result != VDO_SUCCESS) { + vdo_free_recovery_journal(journal); + return result; + } + + result = create_metadata_vio(vdo, VIO_TYPE_RECOVERY_JOURNAL, VIO_PRIORITY_HIGH, + journal, NULL, &journal->flush_vio); + if (result != VDO_SUCCESS) { + vdo_free_recovery_journal(journal); + return result; + } + + result = vdo_register_read_only_listener(vdo, journal, + notify_recovery_journal_of_read_only_mode, + journal->thread_id); + if (result != VDO_SUCCESS) { + vdo_free_recovery_journal(journal); + return result; + } + + result = vdo_make_default_thread(vdo, journal->thread_id); + if (result != VDO_SUCCESS) { + vdo_free_recovery_journal(journal); + return result; + } + + journal->flush_vio->completion.callback_thread_id = journal->thread_id; + *journal_ptr = journal; + return VDO_SUCCESS; +} + +/** + * vdo_free_recovery_journal() - Free a recovery journal. + * @journal: The recovery journal to free. + */ +void vdo_free_recovery_journal(struct recovery_journal *journal) +{ + block_count_t i; + + if (journal == NULL) + return; + + vdo_free(vdo_forget(journal->lock_counter.logical_zone_counts)); + vdo_free(vdo_forget(journal->lock_counter.physical_zone_counts)); + vdo_free(vdo_forget(journal->lock_counter.journal_counters)); + vdo_free(vdo_forget(journal->lock_counter.journal_decrement_counts)); + vdo_free(vdo_forget(journal->lock_counter.logical_counters)); + vdo_free(vdo_forget(journal->lock_counter.physical_counters)); + free_vio(vdo_forget(journal->flush_vio)); + + /* + * FIXME: eventually, the journal should be constructed in a quiescent state which + * requires opening before use. + */ + if (!vdo_is_state_quiescent(&journal->state)) { + VDO_ASSERT_LOG_ONLY(list_empty(&journal->active_tail_blocks), + "journal being freed has no active tail blocks"); + } else if (!vdo_is_state_saved(&journal->state) && + !list_empty(&journal->active_tail_blocks)) { + vdo_log_warning("journal being freed has uncommitted entries"); + } + + for (i = 0; i < RECOVERY_JOURNAL_RESERVED_BLOCKS; i++) { + struct recovery_journal_block *block = &journal->blocks[i]; + + vdo_free(vdo_forget(block->vio.data)); + free_vio_components(&block->vio); + } + + vdo_free(journal); +} + +/** + * vdo_initialize_recovery_journal_post_repair() - Initialize the journal after a repair. + * @journal: The journal in question. + * @recovery_count: The number of completed recoveries. + * @tail: The new tail block sequence number. + * @logical_blocks_used: The new number of logical blocks used. + * @block_map_data_blocks: The new number of block map data blocks. + */ +void vdo_initialize_recovery_journal_post_repair(struct recovery_journal *journal, + u64 recovery_count, + sequence_number_t tail, + block_count_t logical_blocks_used, + block_count_t block_map_data_blocks) +{ + set_journal_tail(journal, tail + 1); + journal->recovery_count = compute_recovery_count_byte(recovery_count); + initialize_journal_state(journal); + journal->logical_blocks_used = logical_blocks_used; + journal->block_map_data_blocks = block_map_data_blocks; +} + +/** + * vdo_get_journal_block_map_data_blocks_used() - Get the number of block map pages, allocated from + * data blocks, currently in use. + * @journal: The journal in question. + * + * Return: The number of block map pages allocated from slabs. + */ +block_count_t vdo_get_journal_block_map_data_blocks_used(struct recovery_journal *journal) +{ + return journal->block_map_data_blocks; +} + +/** + * vdo_get_recovery_journal_thread_id() - Get the ID of a recovery journal's thread. + * @journal: The journal to query. + * + * Return: The ID of the journal's thread. + */ +thread_id_t vdo_get_recovery_journal_thread_id(struct recovery_journal *journal) +{ + return journal->thread_id; +} + +/** + * vdo_open_recovery_journal() - Prepare the journal for new entries. + * @journal: The journal in question. + * @depot: The slab depot for this VDO. + * @block_map: The block map for this VDO. + */ +void vdo_open_recovery_journal(struct recovery_journal *journal, + struct slab_depot *depot, struct block_map *block_map) +{ + journal->depot = depot; + journal->block_map = block_map; + WRITE_ONCE(journal->state.current_state, VDO_ADMIN_STATE_NORMAL_OPERATION); +} + +/** + * vdo_record_recovery_journal() - Record the state of a recovery journal for encoding in the super + * block. + * @journal: the recovery journal. + * + * Return: the state of the journal. + */ +struct recovery_journal_state_7_0 +vdo_record_recovery_journal(const struct recovery_journal *journal) +{ + struct recovery_journal_state_7_0 state = { + .logical_blocks_used = journal->logical_blocks_used, + .block_map_data_blocks = journal->block_map_data_blocks, + }; + + if (vdo_is_state_saved(&journal->state)) { + /* + * If the journal is saved, we should start one past the active block (since the + * active block is not guaranteed to be empty). + */ + state.journal_start = journal->tail; + } else { + /* + * When we're merely suspended or have gone read-only, we must record the first + * block that might have entries that need to be applied. + */ + state.journal_start = get_recovery_journal_head(journal); + } + + return state; +} + +/** + * get_block_header() - Get a pointer to the packed journal block header in the block buffer. + * @block: The recovery block. + * + * Return: The block's header. + */ +static inline struct packed_journal_header * +get_block_header(const struct recovery_journal_block *block) +{ + return (struct packed_journal_header *) block->vio.data; +} + +/** + * set_active_sector() - Set the current sector of the current block and initialize it. + * @block: The block to update. + * @sector: A pointer to the first byte of the new sector. + */ +static void set_active_sector(struct recovery_journal_block *block, void *sector) +{ + block->sector = sector; + block->sector->check_byte = get_block_header(block)->check_byte; + block->sector->recovery_count = block->journal->recovery_count; + block->sector->entry_count = 0; +} + +/** + * advance_tail() - Advance the tail of the journal. + * @journal: The journal whose tail should be advanced. + * + * Return: true if the tail was advanced. + */ +static bool advance_tail(struct recovery_journal *journal) +{ + struct recovery_block_header unpacked; + struct packed_journal_header *header; + struct recovery_journal_block *block; + + block = journal->active_block = pop_free_list(journal); + if (block == NULL) + return false; + + list_move_tail(&block->list_node, &journal->active_tail_blocks); + + unpacked = (struct recovery_block_header) { + .metadata_type = VDO_METADATA_RECOVERY_JOURNAL_2, + .block_map_data_blocks = journal->block_map_data_blocks, + .logical_blocks_used = journal->logical_blocks_used, + .nonce = journal->nonce, + .recovery_count = journal->recovery_count, + .sequence_number = journal->tail, + .check_byte = vdo_compute_recovery_journal_check_byte(journal, + journal->tail), + }; + + header = get_block_header(block); + memset(block->vio.data, 0x0, VDO_BLOCK_SIZE); + block->sequence_number = journal->tail; + block->entry_count = 0; + block->uncommitted_entry_count = 0; + block->block_number = vdo_get_recovery_journal_block_number(journal, + journal->tail); + + vdo_pack_recovery_block_header(&unpacked, header); + set_active_sector(block, vdo_get_journal_block_sector(header, 1)); + set_journal_tail(journal, journal->tail + 1); + vdo_advance_block_map_era(journal->block_map, journal->tail); + return true; +} + +/** + * initialize_lock_count() - Initialize the value of the journal zone's counter for a given lock. + * @journal: The recovery journal. + * + * Context: This must be called from the journal zone. + */ +static void initialize_lock_count(struct recovery_journal *journal) +{ + u16 *journal_value; + block_count_t lock_number = journal->active_block->block_number; + atomic_t *decrement_counter = get_decrement_counter(journal, lock_number); + + journal_value = get_counter(journal, lock_number, VDO_ZONE_TYPE_JOURNAL, 0); + VDO_ASSERT_LOG_ONLY((*journal_value == atomic_read(decrement_counter)), + "count to be initialized not in use"); + *journal_value = journal->entries_per_block + 1; + atomic_set(decrement_counter, 0); +} + +/** + * prepare_to_assign_entry() - Prepare the currently active block to receive an entry and check + * whether an entry of the given type may be assigned at this time. + * @journal: The journal receiving an entry. + * + * Return: true if there is space in the journal to store an entry of the specified type. + */ +static bool prepare_to_assign_entry(struct recovery_journal *journal) +{ + if (journal->available_space == 0) + return false; + + if (is_block_full(journal->active_block) && !advance_tail(journal)) + return false; + + if (!is_block_empty(journal->active_block)) + return true; + + if ((journal->tail - get_recovery_journal_head(journal)) > journal->size) { + /* Cannot use this block since the journal is full. */ + journal->events.disk_full++; + return false; + } + + /* + * Don't allow the new block to be reaped until all of its entries have been committed to + * the block map and until the journal block has been fully committed as well. Because the + * block map update is done only after any slab journal entries have been made, the + * per-entry lock for the block map entry serves to protect those as well. + */ + initialize_lock_count(journal); + return true; +} + +static void write_blocks(struct recovery_journal *journal); + +/** + * schedule_block_write() - Queue a block for writing. + * @journal: The journal in question. + * @block: The block which is now ready to write. + * + * The block is expected to be full. If the block is currently writing, this is a noop as the block + * will be queued for writing when the write finishes. The block must not currently be queued for + * writing. + */ +static void schedule_block_write(struct recovery_journal *journal, + struct recovery_journal_block *block) +{ + if (!block->committing) + vdo_waitq_enqueue_waiter(&journal->pending_writes, &block->write_waiter); + /* + * At the end of adding entries, or discovering this partial block is now full and ready to + * rewrite, we will call write_blocks() and write a whole batch. + */ +} + +/** + * release_journal_block_reference() - Release a reference to a journal block. + * @block: The journal block from which to release a reference. + */ +static void release_journal_block_reference(struct recovery_journal_block *block) +{ + vdo_release_recovery_journal_block_reference(block->journal, + block->sequence_number, + VDO_ZONE_TYPE_JOURNAL, 0); +} + +static void update_usages(struct recovery_journal *journal, struct data_vio *data_vio) +{ + if (data_vio->increment_updater.operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) { + journal->block_map_data_blocks++; + return; + } + + if (data_vio->new_mapped.state != VDO_MAPPING_STATE_UNMAPPED) + journal->logical_blocks_used++; + + if (data_vio->mapped.state != VDO_MAPPING_STATE_UNMAPPED) + journal->logical_blocks_used--; +} + +/** + * assign_entry() - Assign an entry waiter to the active block. + * + * Implements waiter_callback_fn. + */ +static void assign_entry(struct vdo_waiter *waiter, void *context) +{ + struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); + struct recovery_journal_block *block = context; + struct recovery_journal *journal = block->journal; + + /* Record the point at which we will make the journal entry. */ + data_vio->recovery_journal_point = (struct journal_point) { + .sequence_number = block->sequence_number, + .entry_count = block->entry_count, + }; + + update_usages(journal, data_vio); + journal->available_space--; + + if (!vdo_waitq_has_waiters(&block->entry_waiters)) + journal->events.blocks.started++; + + vdo_waitq_enqueue_waiter(&block->entry_waiters, &data_vio->waiter); + block->entry_count++; + block->uncommitted_entry_count++; + journal->events.entries.started++; + + if (is_block_full(block)) { + /* + * The block is full, so we can write it anytime henceforth. If it is already + * committing, we'll queue it for writing when it comes back. + */ + schedule_block_write(journal, block); + } + + /* Force out slab journal tail blocks when threshold is reached. */ + check_slab_journal_commit_threshold(journal); +} + +static void assign_entries(struct recovery_journal *journal) +{ + if (journal->adding_entries) { + /* Protect against re-entrancy. */ + return; + } + + journal->adding_entries = true; + while (vdo_waitq_has_waiters(&journal->entry_waiters) && + prepare_to_assign_entry(journal)) { + vdo_waitq_notify_next_waiter(&journal->entry_waiters, + assign_entry, journal->active_block); + } + + /* Now that we've finished with entries, see if we have a batch of blocks to write. */ + write_blocks(journal); + journal->adding_entries = false; +} + +/** + * recycle_journal_block() - Prepare an in-memory journal block to be reused now that it has been + * fully committed. + * @block: The block to be recycled. + */ +static void recycle_journal_block(struct recovery_journal_block *block) +{ + struct recovery_journal *journal = block->journal; + block_count_t i; + + list_move_tail(&block->list_node, &journal->free_tail_blocks); + + /* Release any unused entry locks. */ + for (i = block->entry_count; i < journal->entries_per_block; i++) + release_journal_block_reference(block); + + /* + * Release our own lock against reaping now that the block is completely committed, or + * we're giving up because we're in read-only mode. + */ + if (block->entry_count > 0) + release_journal_block_reference(block); + + if (block == journal->active_block) + journal->active_block = NULL; +} + +/** + * continue_committed_waiter() - invoked whenever a VIO is to be released from the journal because + * its entry was committed to disk. + * + * Implements waiter_callback_fn. + */ +static void continue_committed_waiter(struct vdo_waiter *waiter, void *context) +{ + struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter); + struct recovery_journal *journal = context; + int result = (is_read_only(journal) ? VDO_READ_ONLY : VDO_SUCCESS); + bool has_decrement; + + VDO_ASSERT_LOG_ONLY(vdo_before_journal_point(&journal->commit_point, + &data_vio->recovery_journal_point), + "DataVIOs released from recovery journal in order. Recovery journal point is (%llu, %u), but commit waiter point is (%llu, %u)", + (unsigned long long) journal->commit_point.sequence_number, + journal->commit_point.entry_count, + (unsigned long long) data_vio->recovery_journal_point.sequence_number, + data_vio->recovery_journal_point.entry_count); + + journal->commit_point = data_vio->recovery_journal_point; + data_vio->last_async_operation = VIO_ASYNC_OP_UPDATE_REFERENCE_COUNTS; + if (result != VDO_SUCCESS) { + continue_data_vio_with_error(data_vio, result); + return; + } + + /* + * The increment must be launched first since it must come before the + * decrement if they are in the same slab. + */ + has_decrement = (data_vio->decrement_updater.zpbn.pbn != VDO_ZERO_BLOCK); + if ((data_vio->increment_updater.zpbn.pbn != VDO_ZERO_BLOCK) || !has_decrement) + continue_data_vio(data_vio); + + if (has_decrement) + vdo_launch_completion(&data_vio->decrement_completion); +} + +/** + * notify_commit_waiters() - Notify any VIOs whose entries have now committed. + * @journal: The recovery journal to update. + */ +static void notify_commit_waiters(struct recovery_journal *journal) +{ + struct recovery_journal_block *block; + + list_for_each_entry(block, &journal->active_tail_blocks, list_node) { + if (block->committing) + return; + + vdo_waitq_notify_all_waiters(&block->commit_waiters, + continue_committed_waiter, journal); + if (is_read_only(journal)) { + vdo_waitq_notify_all_waiters(&block->entry_waiters, + continue_committed_waiter, + journal); + } else if (is_block_dirty(block) || !is_block_full(block)) { + /* Stop at partially-committed or partially-filled blocks. */ + return; + } + } +} + +/** + * recycle_journal_blocks() - Recycle any journal blocks which have been fully committed. + * @journal: The recovery journal to update. + */ +static void recycle_journal_blocks(struct recovery_journal *journal) +{ + struct recovery_journal_block *block, *tmp; + + list_for_each_entry_safe(block, tmp, &journal->active_tail_blocks, list_node) { + if (block->committing) { + /* Don't recycle committing blocks. */ + return; + } + + if (!is_read_only(journal) && + (is_block_dirty(block) || !is_block_full(block))) { + /* + * Don't recycle partially written or partially full blocks, except in + * read-only mode. + */ + return; + } + + recycle_journal_block(block); + } +} + +/** + * complete_write() - Handle post-commit processing. + * @completion: The completion of the VIO writing this block. + * + * This is the callback registered by write_block(). If more entries accumulated in the block being + * committed while the commit was in progress, another commit will be initiated. + */ +static void complete_write(struct vdo_completion *completion) +{ + struct recovery_journal_block *block = completion->parent; + struct recovery_journal *journal = block->journal; + struct recovery_journal_block *last_active_block; + + assert_on_journal_thread(journal, __func__); + + journal->pending_write_count -= 1; + journal->events.blocks.committed += 1; + journal->events.entries.committed += block->entries_in_commit; + block->uncommitted_entry_count -= block->entries_in_commit; + block->entries_in_commit = 0; + block->committing = false; + + /* If this block is the latest block to be acknowledged, record that fact. */ + if (block->sequence_number > journal->last_write_acknowledged) + journal->last_write_acknowledged = block->sequence_number; + + last_active_block = get_journal_block(&journal->active_tail_blocks); + VDO_ASSERT_LOG_ONLY((block->sequence_number >= last_active_block->sequence_number), + "completed journal write is still active"); + + notify_commit_waiters(journal); + + /* + * Is this block now full? Reaping, and adding entries, might have already sent it off for + * rewriting; else, queue it for rewrite. + */ + if (is_block_dirty(block) && is_block_full(block)) + schedule_block_write(journal, block); + + recycle_journal_blocks(journal); + write_blocks(journal); + + check_for_drain_complete(journal); +} + +static void handle_write_error(struct vdo_completion *completion) +{ + struct recovery_journal_block *block = completion->parent; + struct recovery_journal *journal = block->journal; + + vio_record_metadata_io_error(as_vio(completion)); + vdo_log_error_strerror(completion->result, + "cannot write recovery journal block %llu", + (unsigned long long) block->sequence_number); + enter_journal_read_only_mode(journal, completion->result); + complete_write(completion); +} + +static void complete_write_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct recovery_journal_block *block = vio->completion.parent; + struct recovery_journal *journal = block->journal; + + continue_vio_after_io(vio, complete_write, journal->thread_id); +} + +/** + * add_queued_recovery_entries() - Actually add entries from the queue to the given block. + * @block: The journal block. + */ +static void add_queued_recovery_entries(struct recovery_journal_block *block) +{ + while (vdo_waitq_has_waiters(&block->entry_waiters)) { + struct data_vio *data_vio = + vdo_waiter_as_data_vio(vdo_waitq_dequeue_waiter(&block->entry_waiters)); + struct tree_lock *lock = &data_vio->tree_lock; + struct packed_recovery_journal_entry *packed_entry; + struct recovery_journal_entry new_entry; + + if (block->sector->entry_count == RECOVERY_JOURNAL_ENTRIES_PER_SECTOR) + set_active_sector(block, + (char *) block->sector + VDO_SECTOR_SIZE); + + /* Compose and encode the entry. */ + packed_entry = &block->sector->entries[block->sector->entry_count++]; + new_entry = (struct recovery_journal_entry) { + .mapping = { + .pbn = data_vio->increment_updater.zpbn.pbn, + .state = data_vio->increment_updater.zpbn.state, + }, + .unmapping = { + .pbn = data_vio->decrement_updater.zpbn.pbn, + .state = data_vio->decrement_updater.zpbn.state, + }, + .operation = data_vio->increment_updater.operation, + .slot = lock->tree_slots[lock->height].block_map_slot, + }; + *packed_entry = vdo_pack_recovery_journal_entry(&new_entry); + data_vio->recovery_sequence_number = block->sequence_number; + + /* Enqueue the data_vio to wait for its entry to commit. */ + vdo_waitq_enqueue_waiter(&block->commit_waiters, &data_vio->waiter); + } +} + +/** + * write_block() - Issue a block for writing. + * + * Implements waiter_callback_fn. + */ +static void write_block(struct vdo_waiter *waiter, void *context __always_unused) +{ + struct recovery_journal_block *block = + container_of(waiter, struct recovery_journal_block, write_waiter); + struct recovery_journal *journal = block->journal; + struct packed_journal_header *header = get_block_header(block); + + if (block->committing || !vdo_waitq_has_waiters(&block->entry_waiters) || + is_read_only(journal)) + return; + + block->entries_in_commit = vdo_waitq_num_waiters(&block->entry_waiters); + add_queued_recovery_entries(block); + + journal->pending_write_count += 1; + journal->events.blocks.written += 1; + journal->events.entries.written += block->entries_in_commit; + + header->block_map_head = __cpu_to_le64(journal->block_map_head); + header->slab_journal_head = __cpu_to_le64(journal->slab_journal_head); + header->entry_count = __cpu_to_le16(block->entry_count); + + block->committing = true; + + /* + * We must issue a flush and a FUA for every commit. The flush is necessary to ensure that + * the data being referenced is stable. The FUA is necessary to ensure that the journal + * block itself is stable before allowing overwrites of the lbn's previous data. + */ + vdo_submit_metadata_vio(&block->vio, journal->origin + block->block_number, + complete_write_endio, handle_write_error, + REQ_OP_WRITE | REQ_PRIO | REQ_PREFLUSH | REQ_SYNC | REQ_FUA); +} + + +/** + * write_blocks() - Attempt to commit blocks, according to write policy. + * @journal: The recovery journal. + */ +static void write_blocks(struct recovery_journal *journal) +{ + assert_on_journal_thread(journal, __func__); + /* + * We call this function after adding entries to the journal and after finishing a block + * write. Thus, when this function terminates we must either have no VIOs waiting in the + * journal or have some outstanding IO to provide a future wakeup. + * + * We want to only issue full blocks if there are no pending writes. However, if there are + * no outstanding writes and some unwritten entries, we must issue a block, even if it's + * the active block and it isn't full. + */ + if (journal->pending_write_count > 0) + return; + + /* Write all the full blocks. */ + vdo_waitq_notify_all_waiters(&journal->pending_writes, write_block, NULL); + + /* + * Do we need to write the active block? Only if we have no outstanding writes, even after + * issuing all of the full writes. + */ + if ((journal->pending_write_count == 0) && (journal->active_block != NULL)) + write_block(&journal->active_block->write_waiter, NULL); +} + +/** + * vdo_add_recovery_journal_entry() - Add an entry to a recovery journal. + * @journal: The journal in which to make an entry. + * @data_vio: The data_vio for which to add the entry. The entry will be taken + * from the logical and new_mapped fields of the data_vio. The + * data_vio's recovery_sequence_number field will be set to the + * sequence number of the journal block in which the entry was + * made. + * + * This method is asynchronous. The data_vio will not be called back until the entry is committed + * to the on-disk journal. + */ +void vdo_add_recovery_journal_entry(struct recovery_journal *journal, + struct data_vio *data_vio) +{ + assert_on_journal_thread(journal, __func__); + if (!vdo_is_state_normal(&journal->state)) { + continue_data_vio_with_error(data_vio, VDO_INVALID_ADMIN_STATE); + return; + } + + if (is_read_only(journal)) { + continue_data_vio_with_error(data_vio, VDO_READ_ONLY); + return; + } + + VDO_ASSERT_LOG_ONLY(data_vio->recovery_sequence_number == 0, + "journal lock not held for new entry"); + + vdo_advance_journal_point(&journal->append_point, journal->entries_per_block); + vdo_waitq_enqueue_waiter(&journal->entry_waiters, &data_vio->waiter); + assign_entries(journal); +} + +/** + * is_lock_locked() - Check whether a lock is locked for a zone type. + * @journal: The recovery journal. + * @lock_number: The lock to check. + * @zone_type: The type of the zone. + * + * If the recovery journal has a lock on the lock number, both logical and physical zones are + * considered locked. + * + * Return: true if the specified lock has references (is locked). + */ +static bool is_lock_locked(struct recovery_journal *journal, block_count_t lock_number, + enum vdo_zone_type zone_type) +{ + atomic_t *zone_count; + bool locked; + + if (is_journal_zone_locked(journal, lock_number)) + return true; + + zone_count = get_zone_count_ptr(journal, lock_number, zone_type); + locked = (atomic_read(zone_count) != 0); + /* Pairs with implicit barrier in vdo_release_recovery_journal_block_reference() */ + smp_rmb(); + return locked; +} + +/** + * reap_recovery_journal() - Conduct a sweep on a recovery journal to reclaim unreferenced blocks. + * @journal: The recovery journal. + */ +static void reap_recovery_journal(struct recovery_journal *journal) +{ + if (journal->reaping) { + /* + * We already have an outstanding reap in progress. We need to wait for it to + * finish. + */ + return; + } + + if (vdo_is_state_quiescent(&journal->state)) { + /* We are supposed to not do IO. Don't botch it by reaping. */ + return; + } + + /* + * Start reclaiming blocks only when the journal head has no references. Then stop when a + * block is referenced. + */ + while ((journal->block_map_reap_head < journal->last_write_acknowledged) && + !is_lock_locked(journal, journal->block_map_head_block_number, + VDO_ZONE_TYPE_LOGICAL)) { + journal->block_map_reap_head++; + if (++journal->block_map_head_block_number == journal->size) + journal->block_map_head_block_number = 0; + } + + while ((journal->slab_journal_reap_head < journal->last_write_acknowledged) && + !is_lock_locked(journal, journal->slab_journal_head_block_number, + VDO_ZONE_TYPE_PHYSICAL)) { + journal->slab_journal_reap_head++; + if (++journal->slab_journal_head_block_number == journal->size) + journal->slab_journal_head_block_number = 0; + } + + if ((journal->block_map_reap_head == journal->block_map_head) && + (journal->slab_journal_reap_head == journal->slab_journal_head)) { + /* Nothing happened. */ + return; + } + + /* + * If the block map head will advance, we must flush any block map page modified by the + * entries we are reaping. If the slab journal head will advance, we must flush the slab + * summary update covering the slab journal that just released some lock. + */ + journal->reaping = true; + vdo_submit_flush_vio(journal->flush_vio, flush_endio, handle_flush_error); +} + +/** + * vdo_acquire_recovery_journal_block_reference() - Acquire a reference to a recovery journal block + * from somewhere other than the journal itself. + * @journal: The recovery journal. + * @sequence_number: The journal sequence number of the referenced block. + * @zone_type: The type of the zone making the adjustment. + * @zone_id: The ID of the zone making the adjustment. + */ +void vdo_acquire_recovery_journal_block_reference(struct recovery_journal *journal, + sequence_number_t sequence_number, + enum vdo_zone_type zone_type, + zone_count_t zone_id) +{ + block_count_t lock_number; + u16 *current_value; + + if (sequence_number == 0) + return; + + VDO_ASSERT_LOG_ONLY((zone_type != VDO_ZONE_TYPE_JOURNAL), + "invalid lock count increment from journal zone"); + + lock_number = vdo_get_recovery_journal_block_number(journal, sequence_number); + current_value = get_counter(journal, lock_number, zone_type, zone_id); + VDO_ASSERT_LOG_ONLY(*current_value < U16_MAX, + "increment of lock counter must not overflow"); + + if (*current_value == 0) { + /* + * This zone is acquiring this lock for the first time. Extra barriers because this + * was original developed using an atomic add operation that implicitly had them. + */ + smp_mb__before_atomic(); + atomic_inc(get_zone_count_ptr(journal, lock_number, zone_type)); + /* same as before_atomic */ + smp_mb__after_atomic(); + } + + *current_value += 1; +} + +/** + * vdo_release_journal_entry_lock() - Release a single per-entry reference count for a recovery + * journal block. + * @journal: The recovery journal. + * @sequence_number: The journal sequence number of the referenced block. + */ +void vdo_release_journal_entry_lock(struct recovery_journal *journal, + sequence_number_t sequence_number) +{ + block_count_t lock_number; + + if (sequence_number == 0) + return; + + lock_number = vdo_get_recovery_journal_block_number(journal, sequence_number); + /* + * Extra barriers because this was originally developed using an atomic add operation that + * implicitly had them. + */ + smp_mb__before_atomic(); + atomic_inc(get_decrement_counter(journal, lock_number)); + /* same as before_atomic */ + smp_mb__after_atomic(); +} + +/** + * initiate_drain() - Initiate a drain. + * + * Implements vdo_admin_initiator_fn. + */ +static void initiate_drain(struct admin_state *state) +{ + check_for_drain_complete(container_of(state, struct recovery_journal, state)); +} + +/** + * vdo_drain_recovery_journal() - Drain recovery journal I/O. + * @journal: The journal to drain. + * @operation: The drain operation (suspend or save). + * @parent: The completion to notify once the journal is drained. + * + * All uncommitted entries will be written out. + */ +void vdo_drain_recovery_journal(struct recovery_journal *journal, + const struct admin_state_code *operation, + struct vdo_completion *parent) +{ + assert_on_journal_thread(journal, __func__); + vdo_start_draining(&journal->state, operation, parent, initiate_drain); +} + +/** + * resume_lock_counter() - Re-allow notifications from a suspended lock counter. + * @counter: The counter. + * + * Return: true if the lock counter was suspended. + */ +static bool resume_lock_counter(struct lock_counter *counter) +{ + int prior_state; + + /* + * Extra barriers because this was original developed using a CAS operation that implicitly + * had them. + */ + smp_mb__before_atomic(); + prior_state = atomic_cmpxchg(&counter->state, LOCK_COUNTER_STATE_SUSPENDED, + LOCK_COUNTER_STATE_NOT_NOTIFYING); + /* same as before_atomic */ + smp_mb__after_atomic(); + + return (prior_state == LOCK_COUNTER_STATE_SUSPENDED); +} + +/** + * vdo_resume_recovery_journal() - Resume a recovery journal which has been drained. + * @journal: The journal to resume. + * @parent: The completion to finish once the journal is resumed. + */ +void vdo_resume_recovery_journal(struct recovery_journal *journal, + struct vdo_completion *parent) +{ + bool saved; + + assert_on_journal_thread(journal, __func__); + saved = vdo_is_state_saved(&journal->state); + vdo_set_completion_result(parent, vdo_resume_if_quiescent(&journal->state)); + if (is_read_only(journal)) { + vdo_continue_completion(parent, VDO_READ_ONLY); + return; + } + + if (saved) + initialize_journal_state(journal); + + if (resume_lock_counter(&journal->lock_counter)) { + /* We might have missed a notification. */ + reap_recovery_journal(journal); + } + + vdo_launch_completion(parent); +} + +/** + * vdo_get_recovery_journal_logical_blocks_used() - Get the number of logical blocks in use by the + * VDO. + * @journal: The journal. + * + * Return: The number of logical blocks in use by the VDO. + */ +block_count_t vdo_get_recovery_journal_logical_blocks_used(const struct recovery_journal *journal) +{ + return journal->logical_blocks_used; +} + +/** + * vdo_get_recovery_journal_statistics() - Get the current statistics from the recovery journal. + * @journal: The recovery journal to query. + * + * Return: A copy of the current statistics for the journal. + */ +struct recovery_journal_statistics +vdo_get_recovery_journal_statistics(const struct recovery_journal *journal) +{ + return journal->events; +} + +/** + * dump_recovery_block() - Dump the contents of the recovery block to the log. + * @block: The block to dump. + */ +static void dump_recovery_block(const struct recovery_journal_block *block) +{ + vdo_log_info(" sequence number %llu; entries %u; %s; %zu entry waiters; %zu commit waiters", + (unsigned long long) block->sequence_number, block->entry_count, + (block->committing ? "committing" : "waiting"), + vdo_waitq_num_waiters(&block->entry_waiters), + vdo_waitq_num_waiters(&block->commit_waiters)); +} + +/** + * vdo_dump_recovery_journal_statistics() - Dump some current statistics and other debug info from + * the recovery journal. + * @journal: The recovery journal to dump. + */ +void vdo_dump_recovery_journal_statistics(const struct recovery_journal *journal) +{ + const struct recovery_journal_block *block; + struct recovery_journal_statistics stats = vdo_get_recovery_journal_statistics(journal); + + vdo_log_info("Recovery Journal"); + vdo_log_info(" block_map_head=%llu slab_journal_head=%llu last_write_acknowledged=%llu tail=%llu block_map_reap_head=%llu slab_journal_reap_head=%llu disk_full=%llu slab_journal_commits_requested=%llu entry_waiters=%zu", + (unsigned long long) journal->block_map_head, + (unsigned long long) journal->slab_journal_head, + (unsigned long long) journal->last_write_acknowledged, + (unsigned long long) journal->tail, + (unsigned long long) journal->block_map_reap_head, + (unsigned long long) journal->slab_journal_reap_head, + (unsigned long long) stats.disk_full, + (unsigned long long) stats.slab_journal_commits_requested, + vdo_waitq_num_waiters(&journal->entry_waiters)); + vdo_log_info(" entries: started=%llu written=%llu committed=%llu", + (unsigned long long) stats.entries.started, + (unsigned long long) stats.entries.written, + (unsigned long long) stats.entries.committed); + vdo_log_info(" blocks: started=%llu written=%llu committed=%llu", + (unsigned long long) stats.blocks.started, + (unsigned long long) stats.blocks.written, + (unsigned long long) stats.blocks.committed); + + vdo_log_info(" active blocks:"); + list_for_each_entry(block, &journal->active_tail_blocks, list_node) + dump_recovery_block(block); +} diff --git a/drivers/md/dm-vdo/recovery-journal.h b/drivers/md/dm-vdo/recovery-journal.h new file mode 100644 index 000000000000..899071173015 --- /dev/null +++ b/drivers/md/dm-vdo/recovery-journal.h @@ -0,0 +1,316 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_RECOVERY_JOURNAL_H +#define VDO_RECOVERY_JOURNAL_H + +#include + +#include "numeric.h" + +#include "admin-state.h" +#include "constants.h" +#include "encodings.h" +#include "flush.h" +#include "statistics.h" +#include "types.h" +#include "wait-queue.h" + +/** + * DOC: recovery journal. + * + * The recovery_journal provides a log of all block mapping and reference count changes which have + * not yet been stably written to the block map or slab journals. This log helps to reduce the + * write amplification of writes by providing amortization of slab journal and block map page + * updates. + * + * The recovery journal has a single dedicated queue and thread for performing all journal updates. + * The concurrency guarantees of this single-threaded model allow the code to omit more + * fine-grained locking for recovery journal structures. + * + * The journal consists of a set of on-disk blocks arranged as a circular log with monotonically + * increasing sequence numbers. Three sequence numbers serve to define the active extent of the + * journal. The 'head' is the oldest active block in the journal. The 'tail' is the end of the + * half-open interval containing the active blocks. 'active' is the number of the block actively + * receiving entries. In an empty journal, head == active == tail. Once any entries are added, tail + * = active + 1, and head may be any value in the interval [tail - size, active]. + * + * The journal also contains a set of in-memory blocks which are used to buffer up entries until + * they can be committed. In general the number of in-memory blocks ('tail_buffer_count') will be + * less than the on-disk size. Each in-memory block is also a vdo_completion. Each in-memory block + * has a vio which is used to commit that block to disk. The vio's data is the on-disk + * representation of the journal block. In addition each in-memory block has a buffer which is used + * to accumulate entries while a partial commit of the block is in progress. In-memory blocks are + * kept on two rings. Free blocks live on the 'free_tail_blocks' ring. When a block becomes active + * (see below) it is moved to the 'active_tail_blocks' ring. When a block is fully committed, it is + * moved back to the 'free_tail_blocks' ring. + * + * When entries are added to the journal, they are added to the active in-memory block, as + * indicated by the 'active_block' field. If the caller wishes to wait for the entry to be + * committed, the requesting VIO will be attached to the in-memory block to which the caller's + * entry was added. If the caller does wish to wait, or if the entry filled the active block, an + * attempt will be made to commit that block to disk. If there is already another commit in + * progress, the attempt will be ignored and then automatically retried when the in-progress commit + * completes. If there is no commit in progress, any data_vios waiting on the block are transferred + * to the block's vio which is then written, automatically waking all of the waiters when it + * completes. When the write completes, any entries which accumulated in the block are copied to + * the vio's data buffer. + * + * Finally, the journal maintains a set of counters, one for each on disk journal block. These + * counters are used as locks to prevent premature reaping of journal blocks. Each time a new + * sequence number is used, the counter for the corresponding block is incremented. The counter is + * subsequently decremented when that block is filled and then committed for the last time. This + * prevents blocks from being reaped while they are still being updated. The counter is also + * incremented once for each entry added to a block, and decremented once each time the block map + * is updated in memory for that request. This prevents blocks from being reaped while their VIOs + * are still active. Finally, each in-memory block map page tracks the oldest journal block that + * contains entries corresponding to uncommitted updates to that block map page. Each time an + * in-memory block map page is updated, it checks if the journal block for the VIO is earlier than + * the one it references, in which case it increments the count on the earlier journal block and + * decrements the count on the later journal block, maintaining a lock on the oldest journal block + * containing entries for that page. When a block map page has been flushed from the cache, the + * counter for the journal block it references is decremented. Whenever the counter for the head + * block goes to 0, the head is advanced until it comes to a block whose counter is not 0 or until + * it reaches the active block. This is the mechanism for reclaiming journal space on disk. + * + * If there is no in-memory space when a VIO attempts to add an entry, the VIO will be attached to + * the 'commit_completion' and will be woken the next time a full block has committed. If there is + * no on-disk space when a VIO attempts to add an entry, the VIO will be attached to the + * 'reap_completion', and will be woken the next time a journal block is reaped. + */ + +enum vdo_zone_type { + VDO_ZONE_TYPE_ADMIN, + VDO_ZONE_TYPE_JOURNAL, + VDO_ZONE_TYPE_LOGICAL, + VDO_ZONE_TYPE_PHYSICAL, +}; + +struct lock_counter { + /* The completion for notifying the owner of a lock release */ + struct vdo_completion completion; + /* The number of logical zones which may hold locks */ + zone_count_t logical_zones; + /* The number of physical zones which may hold locks */ + zone_count_t physical_zones; + /* The number of locks */ + block_count_t locks; + /* Whether the lock release notification is in flight */ + atomic_t state; + /* The number of logical zones which hold each lock */ + atomic_t *logical_zone_counts; + /* The number of physical zones which hold each lock */ + atomic_t *physical_zone_counts; + /* The per-lock counts for the journal zone */ + u16 *journal_counters; + /* The per-lock decrement counts for the journal zone */ + atomic_t *journal_decrement_counts; + /* The per-zone, per-lock reference counts for logical zones */ + u16 *logical_counters; + /* The per-zone, per-lock reference counts for physical zones */ + u16 *physical_counters; +}; + +struct recovery_journal_block { + /* The doubly linked pointers for the free or active lists */ + struct list_head list_node; + /* The waiter for the pending full block list */ + struct vdo_waiter write_waiter; + /* The journal to which this block belongs */ + struct recovery_journal *journal; + /* A pointer to the current sector in the packed block buffer */ + struct packed_journal_sector *sector; + /* The vio for writing this block */ + struct vio vio; + /* The sequence number for this block */ + sequence_number_t sequence_number; + /* The location of this block in the on-disk journal */ + physical_block_number_t block_number; + /* Whether this block is being committed */ + bool committing; + /* The total number of entries in this block */ + journal_entry_count_t entry_count; + /* The total number of uncommitted entries (queued or committing) */ + journal_entry_count_t uncommitted_entry_count; + /* The number of new entries in the current commit */ + journal_entry_count_t entries_in_commit; + /* The queue of vios which will make entries for the next commit */ + struct vdo_wait_queue entry_waiters; + /* The queue of vios waiting for the current commit */ + struct vdo_wait_queue commit_waiters; +}; + +struct recovery_journal { + /* The thread ID of the journal zone */ + thread_id_t thread_id; + /* The slab depot which can hold locks on this journal */ + struct slab_depot *depot; + /* The block map which can hold locks on this journal */ + struct block_map *block_map; + /* The queue of vios waiting to make entries */ + struct vdo_wait_queue entry_waiters; + /* The number of free entries in the journal */ + u64 available_space; + /* The number of decrement entries which need to be made */ + data_vio_count_t pending_decrement_count; + /* Whether the journal is adding entries from the increment or decrement waiters queues */ + bool adding_entries; + /* The administrative state of the journal */ + struct admin_state state; + /* Whether a reap is in progress */ + bool reaping; + /* The location of the first journal block */ + physical_block_number_t origin; + /* The oldest active block in the journal on disk for block map rebuild */ + sequence_number_t block_map_head; + /* The oldest active block in the journal on disk for slab journal replay */ + sequence_number_t slab_journal_head; + /* The newest block in the journal on disk to which a write has finished */ + sequence_number_t last_write_acknowledged; + /* The end of the half-open interval of the active journal */ + sequence_number_t tail; + /* The point at which the last entry will have been added */ + struct journal_point append_point; + /* The journal point of the vio most recently released from the journal */ + struct journal_point commit_point; + /* The nonce of the VDO */ + nonce_t nonce; + /* The number of recoveries completed by the VDO */ + u8 recovery_count; + /* The number of entries which fit in a single block */ + journal_entry_count_t entries_per_block; + /* Unused in-memory journal blocks */ + struct list_head free_tail_blocks; + /* In-memory journal blocks with records */ + struct list_head active_tail_blocks; + /* A pointer to the active block (the one we are adding entries to now) */ + struct recovery_journal_block *active_block; + /* Journal blocks that need writing */ + struct vdo_wait_queue pending_writes; + /* The new block map reap head after reaping */ + sequence_number_t block_map_reap_head; + /* The head block number for the block map rebuild range */ + block_count_t block_map_head_block_number; + /* The new slab journal reap head after reaping */ + sequence_number_t slab_journal_reap_head; + /* The head block number for the slab journal replay range */ + block_count_t slab_journal_head_block_number; + /* The data-less vio, usable only for flushing */ + struct vio *flush_vio; + /* The number of blocks in the on-disk journal */ + block_count_t size; + /* The number of logical blocks that are in-use */ + block_count_t logical_blocks_used; + /* The number of block map pages that are allocated */ + block_count_t block_map_data_blocks; + /* The number of journal blocks written but not yet acknowledged */ + block_count_t pending_write_count; + /* The threshold at which slab journal tail blocks will be written out */ + block_count_t slab_journal_commit_threshold; + /* Counters for events in the journal that are reported as statistics */ + struct recovery_journal_statistics events; + /* The locks for each on-disk block */ + struct lock_counter lock_counter; + /* The tail blocks */ + struct recovery_journal_block blocks[]; +}; + +/** + * vdo_get_recovery_journal_block_number() - Get the physical block number for a given sequence + * number. + * @journal: The journal. + * @sequence: The sequence number of the desired block. + * + * Return: The block number corresponding to the sequence number. + */ +static inline physical_block_number_t __must_check +vdo_get_recovery_journal_block_number(const struct recovery_journal *journal, + sequence_number_t sequence) +{ + /* + * Since journal size is a power of two, the block number modulus can just be extracted + * from the low-order bits of the sequence. + */ + return vdo_compute_recovery_journal_block_number(journal->size, sequence); +} + +/** + * vdo_compute_recovery_journal_check_byte() - Compute the check byte for a given sequence number. + * @journal: The journal. + * @sequence: The sequence number. + * + * Return: The check byte corresponding to the sequence number. + */ +static inline u8 __must_check +vdo_compute_recovery_journal_check_byte(const struct recovery_journal *journal, + sequence_number_t sequence) +{ + /* The check byte must change with each trip around the journal. */ + return (((sequence / journal->size) & 0x7F) | 0x80); +} + +int __must_check vdo_decode_recovery_journal(struct recovery_journal_state_7_0 state, + nonce_t nonce, struct vdo *vdo, + struct partition *partition, + u64 recovery_count, + block_count_t journal_size, + struct recovery_journal **journal_ptr); + +void vdo_free_recovery_journal(struct recovery_journal *journal); + +void vdo_initialize_recovery_journal_post_repair(struct recovery_journal *journal, + u64 recovery_count, + sequence_number_t tail, + block_count_t logical_blocks_used, + block_count_t block_map_data_blocks); + +block_count_t __must_check +vdo_get_journal_block_map_data_blocks_used(struct recovery_journal *journal); + +thread_id_t __must_check vdo_get_recovery_journal_thread_id(struct recovery_journal *journal); + +void vdo_open_recovery_journal(struct recovery_journal *journal, + struct slab_depot *depot, struct block_map *block_map); + +sequence_number_t +vdo_get_recovery_journal_current_sequence_number(struct recovery_journal *journal); + +block_count_t __must_check vdo_get_recovery_journal_length(block_count_t journal_size); + +struct recovery_journal_state_7_0 __must_check +vdo_record_recovery_journal(const struct recovery_journal *journal); + +void vdo_add_recovery_journal_entry(struct recovery_journal *journal, + struct data_vio *data_vio); + +void vdo_acquire_recovery_journal_block_reference(struct recovery_journal *journal, + sequence_number_t sequence_number, + enum vdo_zone_type zone_type, + zone_count_t zone_id); + +void vdo_release_recovery_journal_block_reference(struct recovery_journal *journal, + sequence_number_t sequence_number, + enum vdo_zone_type zone_type, + zone_count_t zone_id); + +void vdo_release_journal_entry_lock(struct recovery_journal *journal, + sequence_number_t sequence_number); + +void vdo_drain_recovery_journal(struct recovery_journal *journal, + const struct admin_state_code *operation, + struct vdo_completion *parent); + +void vdo_resume_recovery_journal(struct recovery_journal *journal, + struct vdo_completion *parent); + +block_count_t __must_check +vdo_get_recovery_journal_logical_blocks_used(const struct recovery_journal *journal); + +struct recovery_journal_statistics __must_check +vdo_get_recovery_journal_statistics(const struct recovery_journal *journal); + +void vdo_dump_recovery_journal_statistics(const struct recovery_journal *journal); + +#endif /* VDO_RECOVERY_JOURNAL_H */ diff --git a/drivers/md/dm-vdo/repair.c b/drivers/md/dm-vdo/repair.c new file mode 100644 index 000000000000..defc9359f10e --- /dev/null +++ b/drivers/md/dm-vdo/repair.c @@ -0,0 +1,1756 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "repair.h" + +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "block-map.h" +#include "completion.h" +#include "constants.h" +#include "encodings.h" +#include "int-map.h" +#include "io-submitter.h" +#include "recovery-journal.h" +#include "slab-depot.h" +#include "types.h" +#include "vdo.h" +#include "wait-queue.h" + +/* + * An explicitly numbered block mapping. Numbering the mappings allows them to be sorted by logical + * block number during repair while still preserving the relative order of journal entries with + * the same logical block number. + */ +struct numbered_block_mapping { + struct block_map_slot block_map_slot; + struct block_map_entry block_map_entry; + /* A serial number to use during replay */ + u32 number; +} __packed; + +/* + * The absolute position of an entry in the recovery journal, including the sector number and the + * entry number within the sector. + */ +struct recovery_point { + /* Block sequence number */ + sequence_number_t sequence_number; + /* Sector number */ + u8 sector_count; + /* Entry number */ + journal_entry_count_t entry_count; + /* Whether or not the increment portion of the current entry has been applied */ + bool increment_applied; +}; + +struct repair_completion { + /* The completion header */ + struct vdo_completion completion; + + /* A buffer to hold the data read off disk */ + char *journal_data; + + /* For loading the journal */ + data_vio_count_t vio_count; + data_vio_count_t vios_complete; + struct vio *vios; + + /* The number of entries to be applied to the block map */ + size_t block_map_entry_count; + /* The sequence number of the first valid block for block map recovery */ + sequence_number_t block_map_head; + /* The sequence number of the first valid block for slab journal replay */ + sequence_number_t slab_journal_head; + /* The sequence number of the last valid block of the journal (if known) */ + sequence_number_t tail; + /* + * The highest sequence number of the journal. During recovery (vs read-only rebuild), not + * the same as the tail, since the tail ignores blocks after the first hole. + */ + sequence_number_t highest_tail; + + /* The number of logical blocks currently known to be in use */ + block_count_t logical_blocks_used; + /* The number of block map data blocks known to be allocated */ + block_count_t block_map_data_blocks; + + /* These fields are for playing the journal into the block map */ + /* The entry data for the block map recovery */ + struct numbered_block_mapping *entries; + /* The number of entries in the entry array */ + size_t entry_count; + /* number of pending (non-ready) requests*/ + page_count_t outstanding; + /* number of page completions */ + page_count_t page_count; + bool launching; + /* + * a heap wrapping journal_entries. It re-orders and sorts journal entries in ascending LBN + * order, then original journal order. This permits efficient iteration over the journal + * entries in order. + */ + struct min_heap replay_heap; + /* Fields tracking progress through the journal entries. */ + struct numbered_block_mapping *current_entry; + struct numbered_block_mapping *current_unfetched_entry; + /* Current requested page's PBN */ + physical_block_number_t pbn; + + /* These fields are only used during recovery. */ + /* A location just beyond the last valid entry of the journal */ + struct recovery_point tail_recovery_point; + /* The location of the next recovery journal entry to apply */ + struct recovery_point next_recovery_point; + /* The journal point to give to the next synthesized decref */ + struct journal_point next_journal_point; + /* The number of entries played into slab journals */ + size_t entries_added_to_slab_journals; + + /* These fields are only used during read-only rebuild */ + page_count_t page_to_fetch; + /* the number of leaf pages in the block map */ + page_count_t leaf_pages; + /* the last slot of the block map */ + struct block_map_slot last_slot; + + /* + * The page completions used for playing the journal into the block map, and, during + * read-only rebuild, for rebuilding the reference counts from the block map. + */ + struct vdo_page_completion page_completions[]; +}; + +/* + * This is a min_heap callback function that orders numbered_block_mappings using the + * 'block_map_slot' field as the primary key and the mapping 'number' field as the secondary key. + * Using the mapping number preserves the journal order of entries for the same slot, allowing us + * to sort by slot while still ensuring we replay all entries with the same slot in the exact order + * as they appeared in the journal. + */ +static bool mapping_is_less_than(const void *item1, const void *item2) +{ + const struct numbered_block_mapping *mapping1 = + (const struct numbered_block_mapping *) item1; + const struct numbered_block_mapping *mapping2 = + (const struct numbered_block_mapping *) item2; + + if (mapping1->block_map_slot.pbn != mapping2->block_map_slot.pbn) + return mapping1->block_map_slot.pbn < mapping2->block_map_slot.pbn; + + if (mapping1->block_map_slot.slot != mapping2->block_map_slot.slot) + return mapping1->block_map_slot.slot < mapping2->block_map_slot.slot; + + if (mapping1->number != mapping2->number) + return mapping1->number < mapping2->number; + + return 0; +} + +static void swap_mappings(void *item1, void *item2) +{ + struct numbered_block_mapping *mapping1 = item1; + struct numbered_block_mapping *mapping2 = item2; + + swap(*mapping1, *mapping2); +} + +static const struct min_heap_callbacks repair_min_heap = { + .elem_size = sizeof(struct numbered_block_mapping), + .less = mapping_is_less_than, + .swp = swap_mappings, +}; + +static struct numbered_block_mapping *sort_next_heap_element(struct repair_completion *repair) +{ + struct min_heap *heap = &repair->replay_heap; + struct numbered_block_mapping *last; + + if (heap->nr == 0) + return NULL; + + /* + * Swap the next heap element with the last one on the heap, popping it off the heap, + * restore the heap invariant, and return a pointer to the popped element. + */ + last = &repair->entries[--heap->nr]; + swap_mappings(heap->data, last); + min_heapify(heap, 0, &repair_min_heap); + return last; +} + +/** + * as_repair_completion() - Convert a generic completion to a repair_completion. + * @completion: The completion to convert. + * + * Return: The repair_completion. + */ +static inline struct repair_completion * __must_check +as_repair_completion(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_REPAIR_COMPLETION); + return container_of(completion, struct repair_completion, completion); +} + +static void prepare_repair_completion(struct repair_completion *repair, + vdo_action_fn callback, enum vdo_zone_type zone_type) +{ + struct vdo_completion *completion = &repair->completion; + const struct thread_config *thread_config = &completion->vdo->thread_config; + thread_id_t thread_id; + + /* All blockmap access is done on single thread, so use logical zone 0. */ + thread_id = ((zone_type == VDO_ZONE_TYPE_LOGICAL) ? + thread_config->logical_threads[0] : + thread_config->admin_thread); + vdo_reset_completion(completion); + vdo_set_completion_callback(completion, callback, thread_id); +} + +static void launch_repair_completion(struct repair_completion *repair, + vdo_action_fn callback, enum vdo_zone_type zone_type) +{ + prepare_repair_completion(repair, callback, zone_type); + vdo_launch_completion(&repair->completion); +} + +static void uninitialize_vios(struct repair_completion *repair) +{ + while (repair->vio_count > 0) + free_vio_components(&repair->vios[--repair->vio_count]); + + vdo_free(vdo_forget(repair->vios)); +} + +static void free_repair_completion(struct repair_completion *repair) +{ + if (repair == NULL) + return; + + /* + * We do this here because this function is the only common bottleneck for all clean up + * paths. + */ + repair->completion.vdo->block_map->zones[0].page_cache.rebuilding = false; + + uninitialize_vios(repair); + vdo_free(vdo_forget(repair->journal_data)); + vdo_free(vdo_forget(repair->entries)); + vdo_free(repair); +} + +static void finish_repair(struct vdo_completion *completion) +{ + struct vdo_completion *parent = completion->parent; + struct vdo *vdo = completion->vdo; + struct repair_completion *repair = as_repair_completion(completion); + + vdo_assert_on_admin_thread(vdo, __func__); + + if (vdo->load_state != VDO_REBUILD_FOR_UPGRADE) + vdo->states.vdo.complete_recoveries++; + + vdo_initialize_recovery_journal_post_repair(vdo->recovery_journal, + vdo->states.vdo.complete_recoveries, + repair->highest_tail, + repair->logical_blocks_used, + repair->block_map_data_blocks); + free_repair_completion(vdo_forget(repair)); + + if (vdo_state_requires_read_only_rebuild(vdo->load_state)) { + vdo_log_info("Read-only rebuild complete"); + vdo_launch_completion(parent); + return; + } + + /* FIXME: shouldn't this say either "recovery" or "repair"? */ + vdo_log_info("Rebuild complete"); + + /* + * Now that we've freed the repair completion and its vast array of journal entries, we + * can allocate refcounts. + */ + vdo_continue_completion(parent, vdo_allocate_reference_counters(vdo->depot)); +} + +/** + * abort_repair() - Handle a repair error. + * @completion: The repair completion. + */ +static void abort_repair(struct vdo_completion *completion) +{ + struct vdo_completion *parent = completion->parent; + int result = completion->result; + struct repair_completion *repair = as_repair_completion(completion); + + if (vdo_state_requires_read_only_rebuild(completion->vdo->load_state)) + vdo_log_info("Read-only rebuild aborted"); + else + vdo_log_warning("Recovery aborted"); + + free_repair_completion(vdo_forget(repair)); + vdo_continue_completion(parent, result); +} + +/** + * abort_on_error() - Abort a repair if there is an error. + * @result: The result to check. + * @repair: The repair completion. + * + * Return: true if the result was an error. + */ +static bool __must_check abort_on_error(int result, struct repair_completion *repair) +{ + if (result == VDO_SUCCESS) + return false; + + vdo_fail_completion(&repair->completion, result); + return true; +} + +/** + * drain_slab_depot() - Flush out all dirty refcounts blocks now that they have been rebuilt or + * recovered. + */ +static void drain_slab_depot(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + struct repair_completion *repair = as_repair_completion(completion); + const struct admin_state_code *operation; + + vdo_assert_on_admin_thread(vdo, __func__); + + prepare_repair_completion(repair, finish_repair, VDO_ZONE_TYPE_ADMIN); + if (vdo_state_requires_read_only_rebuild(vdo->load_state)) { + vdo_log_info("Saving rebuilt state"); + operation = VDO_ADMIN_STATE_REBUILDING; + } else { + vdo_log_info("Replayed %zu journal entries into slab journals", + repair->entries_added_to_slab_journals); + operation = VDO_ADMIN_STATE_RECOVERING; + } + + vdo_drain_slab_depot(vdo->depot, operation, completion); +} + +/** + * flush_block_map_updates() - Flush the block map now that all the reference counts are rebuilt. + * @completion: The repair completion. + * + * This callback is registered in finish_if_done(). + */ +static void flush_block_map_updates(struct vdo_completion *completion) +{ + vdo_assert_on_admin_thread(completion->vdo, __func__); + + vdo_log_info("Flushing block map changes"); + prepare_repair_completion(as_repair_completion(completion), drain_slab_depot, + VDO_ZONE_TYPE_ADMIN); + vdo_drain_block_map(completion->vdo->block_map, VDO_ADMIN_STATE_RECOVERING, + completion); +} + +static bool fetch_page(struct repair_completion *repair, + struct vdo_completion *completion); + +/** + * handle_page_load_error() - Handle an error loading a page. + * @completion: The vdo_page_completion. + */ +static void handle_page_load_error(struct vdo_completion *completion) +{ + struct repair_completion *repair = completion->parent; + + repair->outstanding--; + vdo_set_completion_result(&repair->completion, completion->result); + vdo_release_page_completion(completion); + fetch_page(repair, completion); +} + +/** + * unmap_entry() - Unmap an invalid entry and indicate that its page must be written out. + * @page: The page containing the entries + * @completion: The page_completion for writing the page + * @slot: The slot to unmap + */ +static void unmap_entry(struct block_map_page *page, struct vdo_completion *completion, + slot_number_t slot) +{ + page->entries[slot] = UNMAPPED_BLOCK_MAP_ENTRY; + vdo_request_page_write(completion); +} + +/** + * remove_out_of_bounds_entries() - Unmap entries which outside the logical space. + * @page: The page containing the entries + * @completion: The page_completion for writing the page + * @start: The first slot to check + */ +static void remove_out_of_bounds_entries(struct block_map_page *page, + struct vdo_completion *completion, + slot_number_t start) +{ + slot_number_t slot; + + for (slot = start; slot < VDO_BLOCK_MAP_ENTRIES_PER_PAGE; slot++) { + struct data_location mapping = vdo_unpack_block_map_entry(&page->entries[slot]); + + if (vdo_is_mapped_location(&mapping)) + unmap_entry(page, completion, slot); + } +} + +/** + * process_slot() - Update the reference counts for a single entry. + * @page: The page containing the entries + * @completion: The page_completion for writing the page + * @slot: The slot to check + * + * Return: true if the entry was a valid mapping + */ +static bool process_slot(struct block_map_page *page, struct vdo_completion *completion, + slot_number_t slot) +{ + struct slab_depot *depot = completion->vdo->depot; + int result; + struct data_location mapping = vdo_unpack_block_map_entry(&page->entries[slot]); + + if (!vdo_is_valid_location(&mapping)) { + /* This entry is invalid, so remove it from the page. */ + unmap_entry(page, completion, slot); + return false; + } + + if (!vdo_is_mapped_location(&mapping)) + return false; + + + if (mapping.pbn == VDO_ZERO_BLOCK) + return true; + + if (!vdo_is_physical_data_block(depot, mapping.pbn)) { + /* + * This is a nonsense mapping. Remove it from the map so we're at least consistent + * and mark the page dirty. + */ + unmap_entry(page, completion, slot); + return false; + } + + result = vdo_adjust_reference_count_for_rebuild(depot, mapping.pbn, + VDO_JOURNAL_DATA_REMAPPING); + if (result == VDO_SUCCESS) + return true; + + vdo_log_error_strerror(result, + "Could not adjust reference count for PBN %llu, slot %u mapped to PBN %llu", + (unsigned long long) vdo_get_block_map_page_pbn(page), + slot, (unsigned long long) mapping.pbn); + unmap_entry(page, completion, slot); + return false; +} + +/** + * rebuild_reference_counts_from_page() - Rebuild reference counts from a block map page. + * @repair: The repair completion. + * @completion: The page completion holding the page. + */ +static void rebuild_reference_counts_from_page(struct repair_completion *repair, + struct vdo_completion *completion) +{ + slot_number_t slot, last_slot; + struct block_map_page *page; + int result; + + result = vdo_get_cached_page(completion, &page); + if (result != VDO_SUCCESS) { + vdo_set_completion_result(&repair->completion, result); + return; + } + + if (!page->header.initialized) + return; + + /* Remove any bogus entries which exist beyond the end of the logical space. */ + if (vdo_get_block_map_page_pbn(page) == repair->last_slot.pbn) { + last_slot = repair->last_slot.slot; + remove_out_of_bounds_entries(page, completion, last_slot); + } else { + last_slot = VDO_BLOCK_MAP_ENTRIES_PER_PAGE; + } + + /* Inform the slab depot of all entries on this page. */ + for (slot = 0; slot < last_slot; slot++) { + if (process_slot(page, completion, slot)) + repair->logical_blocks_used++; + } +} + +/** + * page_loaded() - Process a page which has just been loaded. + * @completion: The vdo_page_completion for the fetched page. + * + * This callback is registered by fetch_page(). + */ +static void page_loaded(struct vdo_completion *completion) +{ + struct repair_completion *repair = completion->parent; + + repair->outstanding--; + rebuild_reference_counts_from_page(repair, completion); + vdo_release_page_completion(completion); + + /* Advance progress to the next page, and fetch the next page we haven't yet requested. */ + fetch_page(repair, completion); +} + +static physical_block_number_t get_pbn_to_fetch(struct repair_completion *repair, + struct block_map *block_map) +{ + physical_block_number_t pbn = VDO_ZERO_BLOCK; + + if (repair->completion.result != VDO_SUCCESS) + return VDO_ZERO_BLOCK; + + while ((pbn == VDO_ZERO_BLOCK) && (repair->page_to_fetch < repair->leaf_pages)) + pbn = vdo_find_block_map_page_pbn(block_map, repair->page_to_fetch++); + + if (vdo_is_physical_data_block(repair->completion.vdo->depot, pbn)) + return pbn; + + vdo_set_completion_result(&repair->completion, VDO_BAD_MAPPING); + return VDO_ZERO_BLOCK; +} + +/** + * fetch_page() - Fetch a page from the block map. + * @repair: The repair_completion. + * @completion: The page completion to use. + * + * Return true if the rebuild is complete + */ +static bool fetch_page(struct repair_completion *repair, + struct vdo_completion *completion) +{ + struct vdo_page_completion *page_completion = (struct vdo_page_completion *) completion; + struct block_map *block_map = repair->completion.vdo->block_map; + physical_block_number_t pbn = get_pbn_to_fetch(repair, block_map); + + if (pbn != VDO_ZERO_BLOCK) { + repair->outstanding++; + /* + * We must set the requeue flag here to ensure that we don't blow the stack if all + * the requested pages are already in the cache or get load errors. + */ + vdo_get_page(page_completion, &block_map->zones[0], pbn, true, repair, + page_loaded, handle_page_load_error, true); + } + + if (repair->outstanding > 0) + return false; + + launch_repair_completion(repair, flush_block_map_updates, VDO_ZONE_TYPE_ADMIN); + return true; +} + +/** + * rebuild_from_leaves() - Rebuild reference counts from the leaf block map pages. + * @completion: The repair completion. + * + * Rebuilds reference counts from the leaf block map pages now that reference counts have been + * rebuilt from the interior tree pages (which have been loaded in the process). This callback is + * registered in rebuild_reference_counts(). + */ +static void rebuild_from_leaves(struct vdo_completion *completion) +{ + page_count_t i; + struct repair_completion *repair = as_repair_completion(completion); + struct block_map *map = completion->vdo->block_map; + + repair->logical_blocks_used = 0; + + /* + * The PBN calculation doesn't work until the tree pages have been loaded, so we can't set + * this value at the start of repair. + */ + repair->leaf_pages = vdo_compute_block_map_page_count(map->entry_count); + repair->last_slot = (struct block_map_slot) { + .slot = map->entry_count % VDO_BLOCK_MAP_ENTRIES_PER_PAGE, + .pbn = vdo_find_block_map_page_pbn(map, repair->leaf_pages - 1), + }; + if (repair->last_slot.slot == 0) + repair->last_slot.slot = VDO_BLOCK_MAP_ENTRIES_PER_PAGE; + + for (i = 0; i < repair->page_count; i++) { + if (fetch_page(repair, &repair->page_completions[i].completion)) { + /* + * The rebuild has already moved on, so it isn't safe nor is there a need + * to launch any more fetches. + */ + return; + } + } +} + +/** + * process_entry() - Process a single entry from the block map tree. + * @pbn: A pbn which holds a block map tree page. + * @completion: The parent completion of the traversal. + * + * Implements vdo_entry_callback_fn. + * + * Return: VDO_SUCCESS or an error. + */ +static int process_entry(physical_block_number_t pbn, struct vdo_completion *completion) +{ + struct repair_completion *repair = as_repair_completion(completion); + struct slab_depot *depot = completion->vdo->depot; + int result; + + if ((pbn == VDO_ZERO_BLOCK) || !vdo_is_physical_data_block(depot, pbn)) { + return vdo_log_error_strerror(VDO_BAD_CONFIGURATION, + "PBN %llu out of range", + (unsigned long long) pbn); + } + + result = vdo_adjust_reference_count_for_rebuild(depot, pbn, + VDO_JOURNAL_BLOCK_MAP_REMAPPING); + if (result != VDO_SUCCESS) { + return vdo_log_error_strerror(result, + "Could not adjust reference count for block map tree PBN %llu", + (unsigned long long) pbn); + } + + repair->block_map_data_blocks++; + return VDO_SUCCESS; +} + +static void rebuild_reference_counts(struct vdo_completion *completion) +{ + struct repair_completion *repair = as_repair_completion(completion); + struct vdo *vdo = completion->vdo; + struct vdo_page_cache *cache = &vdo->block_map->zones[0].page_cache; + + /* We must allocate ref_counts before we can rebuild them. */ + if (abort_on_error(vdo_allocate_reference_counters(vdo->depot), repair)) + return; + + /* + * Completion chaining from page cache hits can lead to stack overflow during the rebuild, + * so clear out the cache before this rebuild phase. + */ + if (abort_on_error(vdo_invalidate_page_cache(cache), repair)) + return; + + prepare_repair_completion(repair, rebuild_from_leaves, VDO_ZONE_TYPE_LOGICAL); + vdo_traverse_forest(vdo->block_map, process_entry, completion); +} + +/** + * increment_recovery_point() - Move the given recovery point forward by one entry. + */ +static void increment_recovery_point(struct recovery_point *point) +{ + if (++point->entry_count < RECOVERY_JOURNAL_ENTRIES_PER_SECTOR) + return; + + point->entry_count = 0; + if (point->sector_count < (VDO_SECTORS_PER_BLOCK - 1)) { + point->sector_count++; + return; + } + + point->sequence_number++; + point->sector_count = 1; +} + +/** + * advance_points() - Advance the current recovery and journal points. + * @repair: The repair_completion whose points are to be advanced. + * @entries_per_block: The number of entries in a recovery journal block. + */ +static void advance_points(struct repair_completion *repair, + journal_entry_count_t entries_per_block) +{ + if (!repair->next_recovery_point.increment_applied) { + repair->next_recovery_point.increment_applied = true; + return; + } + + increment_recovery_point(&repair->next_recovery_point); + vdo_advance_journal_point(&repair->next_journal_point, entries_per_block); + repair->next_recovery_point.increment_applied = false; +} + +/** + * before_recovery_point() - Check whether the first point precedes the second point. + * @first: The first recovery point. + * @second: The second recovery point. + * + * Return: true if the first point precedes the second point. + */ +static bool __must_check before_recovery_point(const struct recovery_point *first, + const struct recovery_point *second) +{ + if (first->sequence_number < second->sequence_number) + return true; + + if (first->sequence_number > second->sequence_number) + return false; + + if (first->sector_count < second->sector_count) + return true; + + return ((first->sector_count == second->sector_count) && + (first->entry_count < second->entry_count)); +} + +static struct packed_journal_sector * __must_check get_sector(struct recovery_journal *journal, + char *journal_data, + sequence_number_t sequence, + u8 sector_number) +{ + off_t offset; + + offset = ((vdo_get_recovery_journal_block_number(journal, sequence) * VDO_BLOCK_SIZE) + + (VDO_SECTOR_SIZE * sector_number)); + return (struct packed_journal_sector *) (journal_data + offset); +} + +/** + * get_entry() - Unpack the recovery journal entry associated with the given recovery point. + * @repair: The repair completion. + * @point: The recovery point. + * + * Return: The unpacked contents of the matching recovery journal entry. + */ +static struct recovery_journal_entry get_entry(const struct repair_completion *repair, + const struct recovery_point *point) +{ + struct packed_journal_sector *sector; + + sector = get_sector(repair->completion.vdo->recovery_journal, + repair->journal_data, point->sequence_number, + point->sector_count); + return vdo_unpack_recovery_journal_entry(§or->entries[point->entry_count]); +} + +/** + * validate_recovery_journal_entry() - Validate a recovery journal entry. + * @vdo: The vdo. + * @entry: The entry to validate. + * + * Return: VDO_SUCCESS or an error. + */ +static int validate_recovery_journal_entry(const struct vdo *vdo, + const struct recovery_journal_entry *entry) +{ + if ((entry->slot.pbn >= vdo->states.vdo.config.physical_blocks) || + (entry->slot.slot >= VDO_BLOCK_MAP_ENTRIES_PER_PAGE) || + !vdo_is_valid_location(&entry->mapping) || + !vdo_is_valid_location(&entry->unmapping) || + !vdo_is_physical_data_block(vdo->depot, entry->mapping.pbn) || + !vdo_is_physical_data_block(vdo->depot, entry->unmapping.pbn)) { + return vdo_log_error_strerror(VDO_CORRUPT_JOURNAL, + "Invalid entry: %s (%llu, %u) from %llu to %llu is not within bounds", + vdo_get_journal_operation_name(entry->operation), + (unsigned long long) entry->slot.pbn, + entry->slot.slot, + (unsigned long long) entry->unmapping.pbn, + (unsigned long long) entry->mapping.pbn); + } + + if ((entry->operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) && + (vdo_is_state_compressed(entry->mapping.state) || + (entry->mapping.pbn == VDO_ZERO_BLOCK) || + (entry->unmapping.state != VDO_MAPPING_STATE_UNMAPPED) || + (entry->unmapping.pbn != VDO_ZERO_BLOCK))) { + return vdo_log_error_strerror(VDO_CORRUPT_JOURNAL, + "Invalid entry: %s (%llu, %u) from %llu to %llu is not a valid tree mapping", + vdo_get_journal_operation_name(entry->operation), + (unsigned long long) entry->slot.pbn, + entry->slot.slot, + (unsigned long long) entry->unmapping.pbn, + (unsigned long long) entry->mapping.pbn); + } + + return VDO_SUCCESS; +} + +/** + * add_slab_journal_entries() - Replay recovery journal entries into the slab journals of the + * allocator currently being recovered. + * @completion: The allocator completion. + * + * Waits for slab journal tailblock space when necessary. This method is its own callback. + */ +static void add_slab_journal_entries(struct vdo_completion *completion) +{ + struct recovery_point *recovery_point; + struct repair_completion *repair = completion->parent; + struct vdo *vdo = completion->vdo; + struct recovery_journal *journal = vdo->recovery_journal; + struct block_allocator *allocator = vdo_as_block_allocator(completion); + + /* Get ready in case we need to enqueue again. */ + vdo_prepare_completion(completion, add_slab_journal_entries, + vdo_notify_slab_journals_are_recovered, + completion->callback_thread_id, repair); + for (recovery_point = &repair->next_recovery_point; + before_recovery_point(recovery_point, &repair->tail_recovery_point); + advance_points(repair, journal->entries_per_block)) { + int result; + physical_block_number_t pbn; + struct vdo_slab *slab; + struct recovery_journal_entry entry = get_entry(repair, recovery_point); + bool increment = !repair->next_recovery_point.increment_applied; + + if (increment) { + result = validate_recovery_journal_entry(vdo, &entry); + if (result != VDO_SUCCESS) { + vdo_enter_read_only_mode(vdo, result); + vdo_fail_completion(completion, result); + return; + } + + pbn = entry.mapping.pbn; + } else { + pbn = entry.unmapping.pbn; + } + + if (pbn == VDO_ZERO_BLOCK) + continue; + + slab = vdo_get_slab(vdo->depot, pbn); + if (slab->allocator != allocator) + continue; + + if (!vdo_attempt_replay_into_slab(slab, pbn, entry.operation, increment, + &repair->next_journal_point, + completion)) + return; + + repair->entries_added_to_slab_journals++; + } + + vdo_notify_slab_journals_are_recovered(completion); +} + +/** + * vdo_replay_into_slab_journals() - Replay recovery journal entries in the slab journals of slabs + * owned by a given block_allocator. + * @allocator: The allocator whose slab journals are to be recovered. + * @context: The slab depot load context supplied by a recovery when it loads the depot. + */ +void vdo_replay_into_slab_journals(struct block_allocator *allocator, void *context) +{ + struct vdo_completion *completion = &allocator->completion; + struct repair_completion *repair = context; + struct vdo *vdo = completion->vdo; + + vdo_assert_on_physical_zone_thread(vdo, allocator->zone_number, __func__); + if (repair->entry_count == 0) { + /* there's nothing to replay */ + repair->logical_blocks_used = vdo->recovery_journal->logical_blocks_used; + repair->block_map_data_blocks = vdo->recovery_journal->block_map_data_blocks; + vdo_notify_slab_journals_are_recovered(completion); + return; + } + + repair->next_recovery_point = (struct recovery_point) { + .sequence_number = repair->slab_journal_head, + .sector_count = 1, + .entry_count = 0, + }; + + repair->next_journal_point = (struct journal_point) { + .sequence_number = repair->slab_journal_head, + .entry_count = 0, + }; + + vdo_log_info("Replaying entries into slab journals for zone %u", + allocator->zone_number); + completion->parent = repair; + add_slab_journal_entries(completion); +} + +static void load_slab_depot(struct vdo_completion *completion) +{ + struct repair_completion *repair = as_repair_completion(completion); + const struct admin_state_code *operation; + + vdo_assert_on_admin_thread(completion->vdo, __func__); + + if (vdo_state_requires_read_only_rebuild(completion->vdo->load_state)) { + prepare_repair_completion(repair, rebuild_reference_counts, + VDO_ZONE_TYPE_LOGICAL); + operation = VDO_ADMIN_STATE_LOADING_FOR_REBUILD; + } else { + prepare_repair_completion(repair, drain_slab_depot, VDO_ZONE_TYPE_ADMIN); + operation = VDO_ADMIN_STATE_LOADING_FOR_RECOVERY; + } + + vdo_load_slab_depot(completion->vdo->depot, operation, completion, repair); +} + +static void flush_block_map(struct vdo_completion *completion) +{ + struct repair_completion *repair = as_repair_completion(completion); + const struct admin_state_code *operation; + + vdo_assert_on_admin_thread(completion->vdo, __func__); + + vdo_log_info("Flushing block map changes"); + prepare_repair_completion(repair, load_slab_depot, VDO_ZONE_TYPE_ADMIN); + operation = (vdo_state_requires_read_only_rebuild(completion->vdo->load_state) ? + VDO_ADMIN_STATE_REBUILDING : + VDO_ADMIN_STATE_RECOVERING); + vdo_drain_block_map(completion->vdo->block_map, operation, completion); +} + +static bool finish_if_done(struct repair_completion *repair) +{ + /* Pages are still being launched or there is still work to do */ + if (repair->launching || (repair->outstanding > 0)) + return false; + + if (repair->completion.result != VDO_SUCCESS) { + page_count_t i; + + for (i = 0; i < repair->page_count; i++) { + struct vdo_page_completion *page_completion = + &repair->page_completions[i]; + + if (page_completion->ready) + vdo_release_page_completion(&page_completion->completion); + } + + vdo_launch_completion(&repair->completion); + return true; + } + + if (repair->current_entry >= repair->entries) + return false; + + launch_repair_completion(repair, flush_block_map, VDO_ZONE_TYPE_ADMIN); + return true; +} + +static void abort_block_map_recovery(struct repair_completion *repair, int result) +{ + vdo_set_completion_result(&repair->completion, result); + finish_if_done(repair); +} + +/** + * find_entry_starting_next_page() - Find the first journal entry after a given entry which is not + * on the same block map page. + * @current_entry: The entry to search from. + * @needs_sort: Whether sorting is needed to proceed. + * + * Return: Pointer to the first later journal entry on a different block map page, or a pointer to + * just before the journal entries if no subsequent entry is on a different block map page. + */ +static struct numbered_block_mapping * +find_entry_starting_next_page(struct repair_completion *repair, + struct numbered_block_mapping *current_entry, bool needs_sort) +{ + size_t current_page; + + /* If current_entry is invalid, return immediately. */ + if (current_entry < repair->entries) + return current_entry; + + current_page = current_entry->block_map_slot.pbn; + + /* Decrement current_entry until it's out of bounds or on a different page. */ + while ((current_entry >= repair->entries) && + (current_entry->block_map_slot.pbn == current_page)) { + if (needs_sort) { + struct numbered_block_mapping *just_sorted_entry = + sort_next_heap_element(repair); + VDO_ASSERT_LOG_ONLY(just_sorted_entry < current_entry, + "heap is returning elements in an unexpected order"); + } + + current_entry--; + } + + return current_entry; +} + +/* + * Apply a range of journal entries [starting_entry, ending_entry) journal + * entries to a block map page. + */ +static void apply_journal_entries_to_page(struct block_map_page *page, + struct numbered_block_mapping *starting_entry, + struct numbered_block_mapping *ending_entry) +{ + struct numbered_block_mapping *current_entry = starting_entry; + + while (current_entry != ending_entry) { + page->entries[current_entry->block_map_slot.slot] = current_entry->block_map_entry; + current_entry--; + } +} + +static void recover_ready_pages(struct repair_completion *repair, + struct vdo_completion *completion); + +static void block_map_page_loaded(struct vdo_completion *completion) +{ + struct repair_completion *repair = as_repair_completion(completion->parent); + + repair->outstanding--; + if (!repair->launching) + recover_ready_pages(repair, completion); +} + +static void handle_block_map_page_load_error(struct vdo_completion *completion) +{ + struct repair_completion *repair = as_repair_completion(completion->parent); + + repair->outstanding--; + abort_block_map_recovery(repair, completion->result); +} + +static void fetch_block_map_page(struct repair_completion *repair, + struct vdo_completion *completion) +{ + physical_block_number_t pbn; + + if (repair->current_unfetched_entry < repair->entries) + /* Nothing left to fetch. */ + return; + + /* Fetch the next page we haven't yet requested. */ + pbn = repair->current_unfetched_entry->block_map_slot.pbn; + repair->current_unfetched_entry = + find_entry_starting_next_page(repair, repair->current_unfetched_entry, + true); + repair->outstanding++; + vdo_get_page(((struct vdo_page_completion *) completion), + &repair->completion.vdo->block_map->zones[0], pbn, true, + &repair->completion, block_map_page_loaded, + handle_block_map_page_load_error, false); +} + +static struct vdo_page_completion *get_next_page_completion(struct repair_completion *repair, + struct vdo_page_completion *completion) +{ + completion++; + if (completion == (&repair->page_completions[repair->page_count])) + completion = &repair->page_completions[0]; + return completion; +} + +static void recover_ready_pages(struct repair_completion *repair, + struct vdo_completion *completion) +{ + struct vdo_page_completion *page_completion = (struct vdo_page_completion *) completion; + + if (finish_if_done(repair)) + return; + + if (repair->pbn != page_completion->pbn) + return; + + while (page_completion->ready) { + struct numbered_block_mapping *start_of_next_page; + struct block_map_page *page; + int result; + + result = vdo_get_cached_page(completion, &page); + if (result != VDO_SUCCESS) { + abort_block_map_recovery(repair, result); + return; + } + + start_of_next_page = + find_entry_starting_next_page(repair, repair->current_entry, + false); + apply_journal_entries_to_page(page, repair->current_entry, + start_of_next_page); + repair->current_entry = start_of_next_page; + vdo_request_page_write(completion); + vdo_release_page_completion(completion); + + if (finish_if_done(repair)) + return; + + repair->pbn = repair->current_entry->block_map_slot.pbn; + fetch_block_map_page(repair, completion); + page_completion = get_next_page_completion(repair, page_completion); + completion = &page_completion->completion; + } +} + +static void recover_block_map(struct vdo_completion *completion) +{ + struct repair_completion *repair = as_repair_completion(completion); + struct vdo *vdo = completion->vdo; + struct numbered_block_mapping *first_sorted_entry; + page_count_t i; + + vdo_assert_on_logical_zone_thread(vdo, 0, __func__); + + /* Suppress block map errors. */ + vdo->block_map->zones[0].page_cache.rebuilding = + vdo_state_requires_read_only_rebuild(vdo->load_state); + + if (repair->block_map_entry_count == 0) { + vdo_log_info("Replaying 0 recovery entries into block map"); + vdo_free(vdo_forget(repair->journal_data)); + launch_repair_completion(repair, load_slab_depot, VDO_ZONE_TYPE_ADMIN); + return; + } + + /* + * Organize the journal entries into a binary heap so we can iterate over them in sorted + * order incrementally, avoiding an expensive sort call. + */ + repair->replay_heap = (struct min_heap) { + .data = repair->entries, + .nr = repair->block_map_entry_count, + .size = repair->block_map_entry_count, + }; + min_heapify_all(&repair->replay_heap, &repair_min_heap); + + vdo_log_info("Replaying %zu recovery entries into block map", + repair->block_map_entry_count); + + repair->current_entry = &repair->entries[repair->block_map_entry_count - 1]; + first_sorted_entry = sort_next_heap_element(repair); + VDO_ASSERT_LOG_ONLY(first_sorted_entry == repair->current_entry, + "heap is returning elements in an unexpected order"); + + /* Prevent any page from being processed until all pages have been launched. */ + repair->launching = true; + repair->pbn = repair->current_entry->block_map_slot.pbn; + repair->current_unfetched_entry = repair->current_entry; + for (i = 0; i < repair->page_count; i++) { + if (repair->current_unfetched_entry < repair->entries) + break; + + fetch_block_map_page(repair, &repair->page_completions[i].completion); + } + repair->launching = false; + + /* Process any ready pages. */ + recover_ready_pages(repair, &repair->page_completions[0].completion); +} + +/** + * get_recovery_journal_block_header() - Get the block header for a block at a position in the + * journal data and unpack it. + * @journal: The recovery journal. + * @data: The recovery journal data. + * @sequence: The sequence number. + * + * Return: The unpacked header. + */ +static struct recovery_block_header __must_check +get_recovery_journal_block_header(struct recovery_journal *journal, char *data, + sequence_number_t sequence) +{ + physical_block_number_t pbn = + vdo_get_recovery_journal_block_number(journal, sequence); + char *header = &data[pbn * VDO_BLOCK_SIZE]; + + return vdo_unpack_recovery_block_header((struct packed_journal_header *) header); +} + +/** + * is_valid_recovery_journal_block() - Determine whether the given header describes a valid block + * for the given journal. + * @journal: The journal to use. + * @header: The unpacked block header to check. + * @old_ok: Whether an old format header is valid. + * + * A block is not valid if it is unformatted, or if it is older than the last successful recovery + * or reformat. + * + * Return: True if the header is valid. + */ +static bool __must_check is_valid_recovery_journal_block(const struct recovery_journal *journal, + const struct recovery_block_header *header, + bool old_ok) +{ + if ((header->nonce != journal->nonce) || + (header->recovery_count != journal->recovery_count)) + return false; + + if (header->metadata_type == VDO_METADATA_RECOVERY_JOURNAL_2) + return (header->entry_count <= journal->entries_per_block); + + return (old_ok && + (header->metadata_type == VDO_METADATA_RECOVERY_JOURNAL) && + (header->entry_count <= RECOVERY_JOURNAL_1_ENTRIES_PER_BLOCK)); +} + +/** + * is_exact_recovery_journal_block() - Determine whether the given header describes the exact block + * indicated. + * @journal: The journal to use. + * @header: The unpacked block header to check. + * @sequence: The expected sequence number. + * @type: The expected metadata type. + * + * Return: True if the block matches. + */ +static bool __must_check is_exact_recovery_journal_block(const struct recovery_journal *journal, + const struct recovery_block_header *header, + sequence_number_t sequence, + enum vdo_metadata_type type) +{ + return ((header->metadata_type == type) && + (header->sequence_number == sequence) && + (is_valid_recovery_journal_block(journal, header, true))); +} + +/** + * find_recovery_journal_head_and_tail() - Find the tail and head of the journal. + * + * Return: True if there were valid journal blocks. + */ +static bool find_recovery_journal_head_and_tail(struct repair_completion *repair) +{ + struct recovery_journal *journal = repair->completion.vdo->recovery_journal; + bool found_entries = false; + physical_block_number_t i; + + /* + * Ensure that we don't replay old entries since we know the tail recorded in the super + * block must be a lower bound. Not doing so can result in extra data loss by setting the + * tail too early. + */ + repair->highest_tail = journal->tail; + for (i = 0; i < journal->size; i++) { + struct recovery_block_header header = + get_recovery_journal_block_header(journal, repair->journal_data, i); + + if (!is_valid_recovery_journal_block(journal, &header, true)) { + /* This block is old or incorrectly formatted */ + continue; + } + + if (vdo_get_recovery_journal_block_number(journal, header.sequence_number) != i) { + /* This block is in the wrong location */ + continue; + } + + if (header.sequence_number >= repair->highest_tail) { + found_entries = true; + repair->highest_tail = header.sequence_number; + } + + if (!found_entries) + continue; + + if (header.block_map_head > repair->block_map_head) + repair->block_map_head = header.block_map_head; + + if (header.slab_journal_head > repair->slab_journal_head) + repair->slab_journal_head = header.slab_journal_head; + } + + return found_entries; +} + +/** + * unpack_entry() - Unpack a recovery journal entry in either format. + * @vdo: The vdo. + * @packed: The entry to unpack. + * @format: The expected format of the entry. + * @entry: The unpacked entry. + * + * Return: true if the entry should be applied.3 + */ +static bool unpack_entry(struct vdo *vdo, char *packed, enum vdo_metadata_type format, + struct recovery_journal_entry *entry) +{ + if (format == VDO_METADATA_RECOVERY_JOURNAL_2) { + struct packed_recovery_journal_entry *packed_entry = + (struct packed_recovery_journal_entry *) packed; + + *entry = vdo_unpack_recovery_journal_entry(packed_entry); + } else { + physical_block_number_t low32, high4; + + struct packed_recovery_journal_entry_1 *packed_entry = + (struct packed_recovery_journal_entry_1 *) packed; + + if (packed_entry->operation == VDO_JOURNAL_DATA_INCREMENT) + entry->operation = VDO_JOURNAL_DATA_REMAPPING; + else if (packed_entry->operation == VDO_JOURNAL_BLOCK_MAP_INCREMENT) + entry->operation = VDO_JOURNAL_BLOCK_MAP_REMAPPING; + else + return false; + + low32 = __le32_to_cpu(packed_entry->pbn_low_word); + high4 = packed_entry->pbn_high_nibble; + entry->slot = (struct block_map_slot) { + .pbn = ((high4 << 32) | low32), + .slot = (packed_entry->slot_low | (packed_entry->slot_high << 6)), + }; + entry->mapping = vdo_unpack_block_map_entry(&packed_entry->block_map_entry); + entry->unmapping = (struct data_location) { + .pbn = VDO_ZERO_BLOCK, + .state = VDO_MAPPING_STATE_UNMAPPED, + }; + } + + return (validate_recovery_journal_entry(vdo, entry) == VDO_SUCCESS); +} + +/** + * append_sector_entries() - Append an array of recovery journal entries from a journal block + * sector to the array of numbered mappings in the repair completion, + * numbering each entry in the order they are appended. + * @repair: The repair completion. + * @entries: The entries in the sector. + * @format: The format of the sector. + * @entry_count: The number of entries to append. + */ +static void append_sector_entries(struct repair_completion *repair, char *entries, + enum vdo_metadata_type format, + journal_entry_count_t entry_count) +{ + journal_entry_count_t i; + struct vdo *vdo = repair->completion.vdo; + off_t increment = ((format == VDO_METADATA_RECOVERY_JOURNAL_2) + ? sizeof(struct packed_recovery_journal_entry) + : sizeof(struct packed_recovery_journal_entry_1)); + + for (i = 0; i < entry_count; i++, entries += increment) { + struct recovery_journal_entry entry; + + if (!unpack_entry(vdo, entries, format, &entry)) + /* When recovering from read-only mode, ignore damaged entries. */ + continue; + + repair->entries[repair->block_map_entry_count] = + (struct numbered_block_mapping) { + .block_map_slot = entry.slot, + .block_map_entry = vdo_pack_block_map_entry(entry.mapping.pbn, + entry.mapping.state), + .number = repair->block_map_entry_count, + }; + repair->block_map_entry_count++; + } +} + +static journal_entry_count_t entries_per_sector(enum vdo_metadata_type format, + u8 sector_number) +{ + if (format == VDO_METADATA_RECOVERY_JOURNAL_2) + return RECOVERY_JOURNAL_ENTRIES_PER_SECTOR; + + return ((sector_number == (VDO_SECTORS_PER_BLOCK - 1)) + ? RECOVERY_JOURNAL_1_ENTRIES_IN_LAST_SECTOR + : RECOVERY_JOURNAL_1_ENTRIES_PER_SECTOR); +} + +static void extract_entries_from_block(struct repair_completion *repair, + struct recovery_journal *journal, + sequence_number_t sequence, + enum vdo_metadata_type format, + journal_entry_count_t entries) +{ + sector_count_t i; + struct recovery_block_header header = + get_recovery_journal_block_header(journal, repair->journal_data, + sequence); + + if (!is_exact_recovery_journal_block(journal, &header, sequence, format)) { + /* This block is invalid, so skip it. */ + return; + } + + entries = min(entries, header.entry_count); + for (i = 1; i < VDO_SECTORS_PER_BLOCK; i++) { + struct packed_journal_sector *sector = + get_sector(journal, repair->journal_data, sequence, i); + journal_entry_count_t sector_entries = + min(entries, entries_per_sector(format, i)); + + if (vdo_is_valid_recovery_journal_sector(&header, sector, i)) { + /* Only extract as many as the block header calls for. */ + append_sector_entries(repair, (char *) sector->entries, format, + min_t(journal_entry_count_t, + sector->entry_count, + sector_entries)); + } + + /* + * Even if the sector wasn't full, count it as full when counting up to the + * entry count the block header claims. + */ + entries -= sector_entries; + } +} + +static int parse_journal_for_rebuild(struct repair_completion *repair) +{ + int result; + sequence_number_t i; + block_count_t count; + enum vdo_metadata_type format; + struct vdo *vdo = repair->completion.vdo; + struct recovery_journal *journal = vdo->recovery_journal; + journal_entry_count_t entries_per_block = journal->entries_per_block; + + format = get_recovery_journal_block_header(journal, repair->journal_data, + repair->highest_tail).metadata_type; + if (format == VDO_METADATA_RECOVERY_JOURNAL) + entries_per_block = RECOVERY_JOURNAL_1_ENTRIES_PER_BLOCK; + + /* + * Allocate an array of numbered_block_mapping structures large enough to transcribe every + * packed_recovery_journal_entry from every valid journal block. + */ + count = ((repair->highest_tail - repair->block_map_head + 1) * entries_per_block); + result = vdo_allocate(count, struct numbered_block_mapping, __func__, + &repair->entries); + if (result != VDO_SUCCESS) + return result; + + for (i = repair->block_map_head; i <= repair->highest_tail; i++) + extract_entries_from_block(repair, journal, i, format, entries_per_block); + + return VDO_SUCCESS; +} + +static int validate_heads(struct repair_completion *repair) +{ + /* Both reap heads must be behind the tail. */ + if ((repair->block_map_head <= repair->tail) && + (repair->slab_journal_head <= repair->tail)) + return VDO_SUCCESS; + + + return vdo_log_error_strerror(VDO_CORRUPT_JOURNAL, + "Journal tail too early. block map head: %llu, slab journal head: %llu, tail: %llu", + (unsigned long long) repair->block_map_head, + (unsigned long long) repair->slab_journal_head, + (unsigned long long) repair->tail); +} + +/** + * extract_new_mappings() - Find all valid new mappings to be applied to the block map. + * + * The mappings are extracted from the journal and stored in a sortable array so that all of the + * mappings to be applied to a given block map page can be done in a single page fetch. + */ +static int extract_new_mappings(struct repair_completion *repair) +{ + int result; + struct vdo *vdo = repair->completion.vdo; + struct recovery_point recovery_point = { + .sequence_number = repair->block_map_head, + .sector_count = 1, + .entry_count = 0, + }; + + /* + * Allocate an array of numbered_block_mapping structs just large enough to transcribe + * every packed_recovery_journal_entry from every valid journal block. + */ + result = vdo_allocate(repair->entry_count, struct numbered_block_mapping, + __func__, &repair->entries); + if (result != VDO_SUCCESS) + return result; + + for (; before_recovery_point(&recovery_point, &repair->tail_recovery_point); + increment_recovery_point(&recovery_point)) { + struct recovery_journal_entry entry = get_entry(repair, &recovery_point); + + result = validate_recovery_journal_entry(vdo, &entry); + if (result != VDO_SUCCESS) { + vdo_enter_read_only_mode(vdo, result); + return result; + } + + repair->entries[repair->block_map_entry_count] = + (struct numbered_block_mapping) { + .block_map_slot = entry.slot, + .block_map_entry = vdo_pack_block_map_entry(entry.mapping.pbn, + entry.mapping.state), + .number = repair->block_map_entry_count, + }; + repair->block_map_entry_count++; + } + + result = VDO_ASSERT((repair->block_map_entry_count <= repair->entry_count), + "approximate entry count is an upper bound"); + if (result != VDO_SUCCESS) + vdo_enter_read_only_mode(vdo, result); + + return result; +} + +/** + * compute_usages() - Compute the lbns in use and block map data blocks counts from the tail of + * the journal. + */ +static noinline int compute_usages(struct repair_completion *repair) +{ + /* + * This function is declared noinline to avoid a spurious valgrind error regarding the + * following structure being uninitialized. + */ + struct recovery_point recovery_point = { + .sequence_number = repair->tail, + .sector_count = 1, + .entry_count = 0, + }; + + struct vdo *vdo = repair->completion.vdo; + struct recovery_journal *journal = vdo->recovery_journal; + struct recovery_block_header header = + get_recovery_journal_block_header(journal, repair->journal_data, + repair->tail); + + repair->logical_blocks_used = header.logical_blocks_used; + repair->block_map_data_blocks = header.block_map_data_blocks; + + for (; before_recovery_point(&recovery_point, &repair->tail_recovery_point); + increment_recovery_point(&recovery_point)) { + struct recovery_journal_entry entry = get_entry(repair, &recovery_point); + int result; + + result = validate_recovery_journal_entry(vdo, &entry); + if (result != VDO_SUCCESS) { + vdo_enter_read_only_mode(vdo, result); + return result; + } + + if (entry.operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) { + repair->block_map_data_blocks++; + continue; + } + + if (vdo_is_mapped_location(&entry.mapping)) + repair->logical_blocks_used++; + + if (vdo_is_mapped_location(&entry.unmapping)) + repair->logical_blocks_used--; + } + + return VDO_SUCCESS; +} + +static int parse_journal_for_recovery(struct repair_completion *repair) +{ + int result; + sequence_number_t i, head; + bool found_entries = false; + struct recovery_journal *journal = repair->completion.vdo->recovery_journal; + + head = min(repair->block_map_head, repair->slab_journal_head); + for (i = head; i <= repair->highest_tail; i++) { + struct recovery_block_header header; + journal_entry_count_t block_entries; + u8 j; + + repair->tail = i; + repair->tail_recovery_point = (struct recovery_point) { + .sequence_number = i, + .sector_count = 0, + .entry_count = 0, + }; + + header = get_recovery_journal_block_header(journal, repair->journal_data, i); + if (header.metadata_type == VDO_METADATA_RECOVERY_JOURNAL) { + /* This is an old format block, so we need to upgrade */ + vdo_log_error_strerror(VDO_UNSUPPORTED_VERSION, + "Recovery journal is in the old format, a read-only rebuild is required."); + vdo_enter_read_only_mode(repair->completion.vdo, + VDO_UNSUPPORTED_VERSION); + return VDO_UNSUPPORTED_VERSION; + } + + if (!is_exact_recovery_journal_block(journal, &header, i, + VDO_METADATA_RECOVERY_JOURNAL_2)) { + /* A bad block header was found so this must be the end of the journal. */ + break; + } + + block_entries = header.entry_count; + + /* Examine each sector in turn to determine the last valid sector. */ + for (j = 1; j < VDO_SECTORS_PER_BLOCK; j++) { + struct packed_journal_sector *sector = + get_sector(journal, repair->journal_data, i, j); + journal_entry_count_t sector_entries = + min_t(journal_entry_count_t, sector->entry_count, + block_entries); + + /* A bad sector means that this block was torn. */ + if (!vdo_is_valid_recovery_journal_sector(&header, sector, j)) + break; + + if (sector_entries > 0) { + found_entries = true; + repair->tail_recovery_point.sector_count++; + repair->tail_recovery_point.entry_count = sector_entries; + block_entries -= sector_entries; + repair->entry_count += sector_entries; + } + + /* If this sector is short, the later sectors can't matter. */ + if ((sector_entries < RECOVERY_JOURNAL_ENTRIES_PER_SECTOR) || + (block_entries == 0)) + break; + } + + /* If this block was not filled, or if it tore, no later block can matter. */ + if ((header.entry_count != journal->entries_per_block) || (block_entries > 0)) + break; + } + + if (!found_entries) + return validate_heads(repair); + + /* Set the tail to the last valid tail block, if there is one. */ + if (repair->tail_recovery_point.sector_count == 0) + repair->tail--; + + result = validate_heads(repair); + if (result != VDO_SUCCESS) + return result; + + vdo_log_info("Highest-numbered recovery journal block has sequence number %llu, and the highest-numbered usable block is %llu", + (unsigned long long) repair->highest_tail, + (unsigned long long) repair->tail); + + result = extract_new_mappings(repair); + if (result != VDO_SUCCESS) + return result; + + return compute_usages(repair); +} + +static int parse_journal(struct repair_completion *repair) +{ + if (!find_recovery_journal_head_and_tail(repair)) + return VDO_SUCCESS; + + return (vdo_state_requires_read_only_rebuild(repair->completion.vdo->load_state) ? + parse_journal_for_rebuild(repair) : + parse_journal_for_recovery(repair)); +} + +static void finish_journal_load(struct vdo_completion *completion) +{ + struct repair_completion *repair = completion->parent; + + if (++repair->vios_complete != repair->vio_count) + return; + + vdo_log_info("Finished reading recovery journal"); + uninitialize_vios(repair); + prepare_repair_completion(repair, recover_block_map, VDO_ZONE_TYPE_LOGICAL); + vdo_continue_completion(&repair->completion, parse_journal(repair)); +} + +static void handle_journal_load_error(struct vdo_completion *completion) +{ + struct repair_completion *repair = completion->parent; + + /* Preserve the error */ + vdo_set_completion_result(&repair->completion, completion->result); + vio_record_metadata_io_error(as_vio(completion)); + completion->callback(completion); +} + +static void read_journal_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct vdo *vdo = vio->completion.vdo; + + continue_vio_after_io(vio, finish_journal_load, vdo->thread_config.admin_thread); +} + +/** + * vdo_repair() - Load the recovery journal and then recover or rebuild a vdo. + * @parent: The completion to notify when the operation is complete + */ +void vdo_repair(struct vdo_completion *parent) +{ + int result; + char *ptr; + struct repair_completion *repair; + struct vdo *vdo = parent->vdo; + struct recovery_journal *journal = vdo->recovery_journal; + physical_block_number_t pbn = journal->origin; + block_count_t remaining = journal->size; + block_count_t vio_count = DIV_ROUND_UP(remaining, MAX_BLOCKS_PER_VIO); + page_count_t page_count = min_t(page_count_t, + vdo->device_config->cache_size >> 1, + MAXIMUM_SIMULTANEOUS_VDO_BLOCK_MAP_RESTORATION_READS); + + vdo_assert_on_admin_thread(vdo, __func__); + + if (vdo->load_state == VDO_FORCE_REBUILD) { + vdo_log_warning("Rebuilding reference counts to clear read-only mode"); + vdo->states.vdo.read_only_recoveries++; + } else if (vdo->load_state == VDO_REBUILD_FOR_UPGRADE) { + vdo_log_warning("Rebuilding reference counts for upgrade"); + } else { + vdo_log_warning("Device was dirty, rebuilding reference counts"); + } + + result = vdo_allocate_extended(struct repair_completion, page_count, + struct vdo_page_completion, __func__, + &repair); + if (result != VDO_SUCCESS) { + vdo_fail_completion(parent, result); + return; + } + + vdo_initialize_completion(&repair->completion, vdo, VDO_REPAIR_COMPLETION); + repair->completion.error_handler = abort_repair; + repair->completion.parent = parent; + prepare_repair_completion(repair, finish_repair, VDO_ZONE_TYPE_ADMIN); + repair->page_count = page_count; + + result = vdo_allocate(remaining * VDO_BLOCK_SIZE, char, __func__, + &repair->journal_data); + if (abort_on_error(result, repair)) + return; + + result = vdo_allocate(vio_count, struct vio, __func__, &repair->vios); + if (abort_on_error(result, repair)) + return; + + ptr = repair->journal_data; + for (repair->vio_count = 0; repair->vio_count < vio_count; repair->vio_count++) { + block_count_t blocks = min_t(block_count_t, remaining, + MAX_BLOCKS_PER_VIO); + + result = allocate_vio_components(vdo, VIO_TYPE_RECOVERY_JOURNAL, + VIO_PRIORITY_METADATA, + repair, blocks, ptr, + &repair->vios[repair->vio_count]); + if (abort_on_error(result, repair)) + return; + + ptr += (blocks * VDO_BLOCK_SIZE); + remaining -= blocks; + } + + for (vio_count = 0; vio_count < repair->vio_count; + vio_count++, pbn += MAX_BLOCKS_PER_VIO) { + vdo_submit_metadata_vio(&repair->vios[vio_count], pbn, read_journal_endio, + handle_journal_load_error, REQ_OP_READ); + } +} diff --git a/drivers/md/dm-vdo/repair.h b/drivers/md/dm-vdo/repair.h new file mode 100644 index 000000000000..ff255cf41486 --- /dev/null +++ b/drivers/md/dm-vdo/repair.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_REPAIR_H +#define VDO_REPAIR_H + +#include "types.h" + +void vdo_replay_into_slab_journals(struct block_allocator *allocator, void *context); +void vdo_repair(struct vdo_completion *parent); + +#endif /* VDO_REPAIR_H */ diff --git a/drivers/md/dm-vdo/slab-depot.c b/drivers/md/dm-vdo/slab-depot.c new file mode 100644 index 000000000000..46e4721e5b4f --- /dev/null +++ b/drivers/md/dm-vdo/slab-depot.c @@ -0,0 +1,5101 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "slab-depot.h" + +#include +#include +#include +#include +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "numeric.h" +#include "permassert.h" +#include "string-utils.h" + +#include "action-manager.h" +#include "admin-state.h" +#include "completion.h" +#include "constants.h" +#include "data-vio.h" +#include "encodings.h" +#include "io-submitter.h" +#include "physical-zone.h" +#include "priority-table.h" +#include "recovery-journal.h" +#include "repair.h" +#include "status-codes.h" +#include "types.h" +#include "vdo.h" +#include "vio.h" +#include "wait-queue.h" + +static const u64 BYTES_PER_WORD = sizeof(u64); +static const bool NORMAL_OPERATION = true; + +/** + * get_lock() - Get the lock object for a slab journal block by sequence number. + * @journal: vdo_slab journal to retrieve from. + * @sequence_number: Sequence number of the block. + * + * Return: The lock object for the given sequence number. + */ +static inline struct journal_lock * __must_check get_lock(struct slab_journal *journal, + sequence_number_t sequence_number) +{ + return &journal->locks[sequence_number % journal->size]; +} + +static bool is_slab_open(struct vdo_slab *slab) +{ + return (!vdo_is_state_quiescing(&slab->state) && + !vdo_is_state_quiescent(&slab->state)); +} + +/** + * must_make_entries_to_flush() - Check whether there are entry waiters which should delay a flush. + * @journal: The journal to check. + * + * Return: true if there are no entry waiters, or if the slab is unrecovered. + */ +static inline bool __must_check must_make_entries_to_flush(struct slab_journal *journal) +{ + return ((journal->slab->status != VDO_SLAB_REBUILDING) && + vdo_waitq_has_waiters(&journal->entry_waiters)); +} + +/** + * is_reaping() - Check whether a reap is currently in progress. + * @journal: The journal which may be reaping. + * + * Return: true if the journal is reaping. + */ +static inline bool __must_check is_reaping(struct slab_journal *journal) +{ + return (journal->head != journal->unreapable); +} + +/** + * initialize_tail_block() - Initialize tail block as a new block. + * @journal: The journal whose tail block is being initialized. + */ +static void initialize_tail_block(struct slab_journal *journal) +{ + struct slab_journal_block_header *header = &journal->tail_header; + + header->sequence_number = journal->tail; + header->entry_count = 0; + header->has_block_map_increments = false; +} + +/** + * initialize_journal_state() - Set all journal fields appropriately to start journaling. + * @journal: The journal to be reset, based on its tail sequence number. + */ +static void initialize_journal_state(struct slab_journal *journal) +{ + journal->unreapable = journal->head; + journal->reap_lock = get_lock(journal, journal->unreapable); + journal->next_commit = journal->tail; + journal->summarized = journal->last_summarized = journal->tail; + initialize_tail_block(journal); +} + +/** + * block_is_full() - Check whether a journal block is full. + * @journal: The slab journal for the block. + * + * Return: true if the tail block is full. + */ +static bool __must_check block_is_full(struct slab_journal *journal) +{ + journal_entry_count_t count = journal->tail_header.entry_count; + + return (journal->tail_header.has_block_map_increments ? + (journal->full_entries_per_block == count) : + (journal->entries_per_block == count)); +} + +static void add_entries(struct slab_journal *journal); +static void update_tail_block_location(struct slab_journal *journal); +static void release_journal_locks(struct vdo_waiter *waiter, void *context); + +/** + * is_slab_journal_blank() - Check whether a slab's journal is blank. + * + * A slab journal is blank if it has never had any entries recorded in it. + * + * Return: true if the slab's journal has never been modified. + */ +static bool is_slab_journal_blank(const struct vdo_slab *slab) +{ + return ((slab->journal.tail == 1) && + (slab->journal.tail_header.entry_count == 0)); +} + +/** + * mark_slab_journal_dirty() - Put a slab journal on the dirty ring of its allocator in the correct + * order. + * @journal: The journal to be marked dirty. + * @lock: The recovery journal lock held by the slab journal. + */ +static void mark_slab_journal_dirty(struct slab_journal *journal, sequence_number_t lock) +{ + struct slab_journal *dirty_journal; + struct list_head *dirty_list = &journal->slab->allocator->dirty_slab_journals; + + VDO_ASSERT_LOG_ONLY(journal->recovery_lock == 0, "slab journal was clean"); + + journal->recovery_lock = lock; + list_for_each_entry_reverse(dirty_journal, dirty_list, dirty_entry) { + if (dirty_journal->recovery_lock <= journal->recovery_lock) + break; + } + + list_move_tail(&journal->dirty_entry, dirty_journal->dirty_entry.next); +} + +static void mark_slab_journal_clean(struct slab_journal *journal) +{ + journal->recovery_lock = 0; + list_del_init(&journal->dirty_entry); +} + +static void check_if_slab_drained(struct vdo_slab *slab) +{ + bool read_only; + struct slab_journal *journal = &slab->journal; + const struct admin_state_code *code; + + if (!vdo_is_state_draining(&slab->state) || + must_make_entries_to_flush(journal) || + is_reaping(journal) || + journal->waiting_to_commit || + !list_empty(&journal->uncommitted_blocks) || + journal->updating_slab_summary || + (slab->active_count > 0)) + return; + + /* When not suspending or recovering, the slab must be clean. */ + code = vdo_get_admin_state_code(&slab->state); + read_only = vdo_is_read_only(slab->allocator->depot->vdo); + if (!read_only && + vdo_waitq_has_waiters(&slab->dirty_blocks) && + (code != VDO_ADMIN_STATE_SUSPENDING) && + (code != VDO_ADMIN_STATE_RECOVERING)) + return; + + vdo_finish_draining_with_result(&slab->state, + (read_only ? VDO_READ_ONLY : VDO_SUCCESS)); +} + +/* FULLNESS HINT COMPUTATION */ + +/** + * compute_fullness_hint() - Translate a slab's free block count into a 'fullness hint' that can be + * stored in a slab_summary_entry's 7 bits that are dedicated to its free + * count. + * @depot: The depot whose summary being updated. + * @free_blocks: The number of free blocks. + * + * Note: the number of free blocks must be strictly less than 2^23 blocks, even though + * theoretically slabs could contain precisely 2^23 blocks; there is an assumption that at least + * one block is used by metadata. This assumption is necessary; otherwise, the fullness hint might + * overflow. The fullness hint formula is roughly (fullness >> 16) & 0x7f, but (2^23 >> 16) & 0x7f + * is 0, which would make it impossible to distinguish completely full from completely empty. + * + * Return: A fullness hint, which can be stored in 7 bits. + */ +static u8 __must_check compute_fullness_hint(struct slab_depot *depot, + block_count_t free_blocks) +{ + block_count_t hint; + + VDO_ASSERT_LOG_ONLY((free_blocks < (1 << 23)), "free blocks must be less than 2^23"); + + if (free_blocks == 0) + return 0; + + hint = free_blocks >> depot->hint_shift; + return ((hint == 0) ? 1 : hint); +} + +/** + * check_summary_drain_complete() - Check whether an allocators summary has finished draining. + */ +static void check_summary_drain_complete(struct block_allocator *allocator) +{ + if (!vdo_is_state_draining(&allocator->summary_state) || + (allocator->summary_write_count > 0)) + return; + + vdo_finish_operation(&allocator->summary_state, + (vdo_is_read_only(allocator->depot->vdo) ? + VDO_READ_ONLY : VDO_SUCCESS)); +} + +/** + * notify_summary_waiters() - Wake all the waiters in a given queue. + * @allocator: The block allocator summary which owns the queue. + * @queue: The queue to notify. + */ +static void notify_summary_waiters(struct block_allocator *allocator, + struct vdo_wait_queue *queue) +{ + int result = (vdo_is_read_only(allocator->depot->vdo) ? + VDO_READ_ONLY : VDO_SUCCESS); + + vdo_waitq_notify_all_waiters(queue, NULL, &result); +} + +static void launch_write(struct slab_summary_block *summary_block); + +/** + * finish_updating_slab_summary_block() - Finish processing a block which attempted to write, + * whether or not the attempt succeeded. + * @block: The block. + */ +static void finish_updating_slab_summary_block(struct slab_summary_block *block) +{ + notify_summary_waiters(block->allocator, &block->current_update_waiters); + block->writing = false; + block->allocator->summary_write_count--; + if (vdo_waitq_has_waiters(&block->next_update_waiters)) + launch_write(block); + else + check_summary_drain_complete(block->allocator); +} + +/** + * finish_update() - This is the callback for a successful summary block write. + * @completion: The write vio. + */ +static void finish_update(struct vdo_completion *completion) +{ + struct slab_summary_block *block = + container_of(as_vio(completion), struct slab_summary_block, vio); + + atomic64_inc(&block->allocator->depot->summary_statistics.blocks_written); + finish_updating_slab_summary_block(block); +} + +/** + * handle_write_error() - Handle an error writing a slab summary block. + * @completion: The write VIO. + */ +static void handle_write_error(struct vdo_completion *completion) +{ + struct slab_summary_block *block = + container_of(as_vio(completion), struct slab_summary_block, vio); + + vio_record_metadata_io_error(as_vio(completion)); + vdo_enter_read_only_mode(completion->vdo, completion->result); + finish_updating_slab_summary_block(block); +} + +static void write_slab_summary_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct slab_summary_block *block = + container_of(vio, struct slab_summary_block, vio); + + continue_vio_after_io(vio, finish_update, block->allocator->thread_id); +} + +/** + * launch_write() - Write a slab summary block unless it is currently out for writing. + * @block: The block that needs to be committed. + */ +static void launch_write(struct slab_summary_block *block) +{ + struct block_allocator *allocator = block->allocator; + struct slab_depot *depot = allocator->depot; + physical_block_number_t pbn; + + if (block->writing) + return; + + allocator->summary_write_count++; + vdo_waitq_transfer_all_waiters(&block->next_update_waiters, + &block->current_update_waiters); + block->writing = true; + + if (vdo_is_read_only(depot->vdo)) { + finish_updating_slab_summary_block(block); + return; + } + + memcpy(block->outgoing_entries, block->entries, VDO_BLOCK_SIZE); + + /* + * Flush before writing to ensure that the slab journal tail blocks and reference updates + * covered by this summary update are stable. Otherwise, a subsequent recovery could + * encounter a slab summary update that refers to a slab journal tail block that has not + * actually been written. In such cases, the slab journal referenced will be treated as + * empty, causing any data within the slab which predates the existing recovery journal + * entries to be lost. + */ + pbn = (depot->summary_origin + + (VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE * allocator->zone_number) + + block->index); + vdo_submit_metadata_vio(&block->vio, pbn, write_slab_summary_endio, + handle_write_error, REQ_OP_WRITE | REQ_PREFLUSH); +} + +/** + * update_slab_summary_entry() - Update the entry for a slab. + * @slab: The slab whose entry is to be updated + * @waiter: The waiter that is updating the summary. + * @tail_block_offset: The offset of the slab journal's tail block. + * @load_ref_counts: Whether the reference counts must be loaded from disk on the vdo load. + * @is_clean: Whether the slab is clean. + * @free_blocks: The number of free blocks. + */ +static void update_slab_summary_entry(struct vdo_slab *slab, struct vdo_waiter *waiter, + tail_block_offset_t tail_block_offset, + bool load_ref_counts, bool is_clean, + block_count_t free_blocks) +{ + u8 index = slab->slab_number / VDO_SLAB_SUMMARY_ENTRIES_PER_BLOCK; + struct block_allocator *allocator = slab->allocator; + struct slab_summary_block *block = &allocator->summary_blocks[index]; + int result; + struct slab_summary_entry *entry; + + if (vdo_is_read_only(block->vio.completion.vdo)) { + result = VDO_READ_ONLY; + waiter->callback(waiter, &result); + return; + } + + if (vdo_is_state_draining(&allocator->summary_state) || + vdo_is_state_quiescent(&allocator->summary_state)) { + result = VDO_INVALID_ADMIN_STATE; + waiter->callback(waiter, &result); + return; + } + + entry = &allocator->summary_entries[slab->slab_number]; + *entry = (struct slab_summary_entry) { + .tail_block_offset = tail_block_offset, + .load_ref_counts = (entry->load_ref_counts || load_ref_counts), + .is_dirty = !is_clean, + .fullness_hint = compute_fullness_hint(allocator->depot, free_blocks), + }; + vdo_waitq_enqueue_waiter(&block->next_update_waiters, waiter); + launch_write(block); +} + +/** + * finish_reaping() - Actually advance the head of the journal now that any necessary flushes are + * complete. + * @journal: The journal to be reaped. + */ +static void finish_reaping(struct slab_journal *journal) +{ + journal->head = journal->unreapable; + add_entries(journal); + check_if_slab_drained(journal->slab); +} + +static void reap_slab_journal(struct slab_journal *journal); + +/** + * complete_reaping() - Finish reaping now that we have flushed the lower layer and then try + * reaping again in case we deferred reaping due to an outstanding vio. + * @completion: The flush vio. + */ +static void complete_reaping(struct vdo_completion *completion) +{ + struct slab_journal *journal = completion->parent; + + return_vio_to_pool(journal->slab->allocator->vio_pool, + vio_as_pooled_vio(as_vio(vdo_forget(completion)))); + finish_reaping(journal); + reap_slab_journal(journal); +} + +/** + * handle_flush_error() - Handle an error flushing the lower layer. + * @completion: The flush vio. + */ +static void handle_flush_error(struct vdo_completion *completion) +{ + vio_record_metadata_io_error(as_vio(completion)); + vdo_enter_read_only_mode(completion->vdo, completion->result); + complete_reaping(completion); +} + +static void flush_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct slab_journal *journal = vio->completion.parent; + + continue_vio_after_io(vio, complete_reaping, + journal->slab->allocator->thread_id); +} + +/** + * flush_for_reaping() - A waiter callback for getting a vio with which to flush the lower layer + * prior to reaping. + * @waiter: The journal as a flush waiter. + * @context: The newly acquired flush vio. + */ +static void flush_for_reaping(struct vdo_waiter *waiter, void *context) +{ + struct slab_journal *journal = + container_of(waiter, struct slab_journal, flush_waiter); + struct pooled_vio *pooled = context; + struct vio *vio = &pooled->vio; + + vio->completion.parent = journal; + vdo_submit_flush_vio(vio, flush_endio, handle_flush_error); +} + +/** + * reap_slab_journal() - Conduct a reap on a slab journal to reclaim unreferenced blocks. + * @journal: The slab journal. + */ +static void reap_slab_journal(struct slab_journal *journal) +{ + bool reaped = false; + + if (is_reaping(journal)) { + /* We already have a reap in progress so wait for it to finish. */ + return; + } + + if ((journal->slab->status != VDO_SLAB_REBUILT) || + !vdo_is_state_normal(&journal->slab->state) || + vdo_is_read_only(journal->slab->allocator->depot->vdo)) { + /* + * We must not reap in the first two cases, and there's no point in read-only mode. + */ + return; + } + + /* + * Start reclaiming blocks only when the journal head has no references. Then stop when a + * block is referenced or reap reaches the most recently written block, referenced by the + * slab summary, which has the sequence number just before the tail. + */ + while ((journal->unreapable < journal->tail) && (journal->reap_lock->count == 0)) { + reaped = true; + journal->unreapable++; + journal->reap_lock++; + if (journal->reap_lock == &journal->locks[journal->size]) + journal->reap_lock = &journal->locks[0]; + } + + if (!reaped) + return; + + /* + * It is never safe to reap a slab journal block without first issuing a flush, regardless + * of whether a user flush has been received or not. In the absence of the flush, the + * reference block write which released the locks allowing the slab journal to reap may not + * be persisted. Although slab summary writes will eventually issue flushes, multiple slab + * journal block writes can be issued while previous slab summary updates have not yet been + * made. Even though those slab journal block writes will be ignored if the slab summary + * update is not persisted, they may still overwrite the to-be-reaped slab journal block + * resulting in a loss of reference count updates. + */ + journal->flush_waiter.callback = flush_for_reaping; + acquire_vio_from_pool(journal->slab->allocator->vio_pool, + &journal->flush_waiter); +} + +/** + * adjust_slab_journal_block_reference() - Adjust the reference count for a slab journal block. + * @journal: The slab journal. + * @sequence_number: The journal sequence number of the referenced block. + * @adjustment: Amount to adjust the reference counter. + * + * Note that when the adjustment is negative, the slab journal will be reaped. + */ +static void adjust_slab_journal_block_reference(struct slab_journal *journal, + sequence_number_t sequence_number, + int adjustment) +{ + struct journal_lock *lock; + + if (sequence_number == 0) + return; + + if (journal->slab->status == VDO_SLAB_REPLAYING) { + /* Locks should not be used during offline replay. */ + return; + } + + VDO_ASSERT_LOG_ONLY((adjustment != 0), "adjustment must be non-zero"); + lock = get_lock(journal, sequence_number); + if (adjustment < 0) { + VDO_ASSERT_LOG_ONLY((-adjustment <= lock->count), + "adjustment %d of lock count %u for slab journal block %llu must not underflow", + adjustment, lock->count, + (unsigned long long) sequence_number); + } + + lock->count += adjustment; + if (lock->count == 0) + reap_slab_journal(journal); +} + +/** + * release_journal_locks() - Callback invoked after a slab summary update completes. + * @waiter: The slab summary waiter that has just been notified. + * @context: The result code of the update. + * + * Registered in the constructor on behalf of update_tail_block_location(). + * + * Implements waiter_callback_fn. + */ +static void release_journal_locks(struct vdo_waiter *waiter, void *context) +{ + sequence_number_t first, i; + struct slab_journal *journal = + container_of(waiter, struct slab_journal, slab_summary_waiter); + int result = *((int *) context); + + if (result != VDO_SUCCESS) { + if (result != VDO_READ_ONLY) { + /* + * Don't bother logging what might be lots of errors if we are already in + * read-only mode. + */ + vdo_log_error_strerror(result, "failed slab summary update %llu", + (unsigned long long) journal->summarized); + } + + journal->updating_slab_summary = false; + vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo, result); + check_if_slab_drained(journal->slab); + return; + } + + if (journal->partial_write_in_progress && (journal->summarized == journal->tail)) { + journal->partial_write_in_progress = false; + add_entries(journal); + } + + first = journal->last_summarized; + journal->last_summarized = journal->summarized; + for (i = journal->summarized - 1; i >= first; i--) { + /* + * Release the lock the summarized block held on the recovery journal. (During + * replay, recovery_start will always be 0.) + */ + if (journal->recovery_journal != NULL) { + zone_count_t zone_number = journal->slab->allocator->zone_number; + struct journal_lock *lock = get_lock(journal, i); + + vdo_release_recovery_journal_block_reference(journal->recovery_journal, + lock->recovery_start, + VDO_ZONE_TYPE_PHYSICAL, + zone_number); + } + + /* + * Release our own lock against reaping for blocks that are committed. (This + * function will not change locks during replay.) + */ + adjust_slab_journal_block_reference(journal, i, -1); + } + + journal->updating_slab_summary = false; + + reap_slab_journal(journal); + + /* Check if the slab summary needs to be updated again. */ + update_tail_block_location(journal); +} + +/** + * update_tail_block_location() - Update the tail block location in the slab summary, if necessary. + * @journal: The slab journal that is updating its tail block location. + */ +static void update_tail_block_location(struct slab_journal *journal) +{ + block_count_t free_block_count; + struct vdo_slab *slab = journal->slab; + + if (journal->updating_slab_summary || + vdo_is_read_only(journal->slab->allocator->depot->vdo) || + (journal->last_summarized >= journal->next_commit)) { + check_if_slab_drained(slab); + return; + } + + if (slab->status != VDO_SLAB_REBUILT) { + u8 hint = slab->allocator->summary_entries[slab->slab_number].fullness_hint; + + free_block_count = ((block_count_t) hint) << slab->allocator->depot->hint_shift; + } else { + free_block_count = slab->free_blocks; + } + + journal->summarized = journal->next_commit; + journal->updating_slab_summary = true; + + /* + * Update slab summary as dirty. + * vdo_slab journal can only reap past sequence number 1 when all the ref counts for this + * slab have been written to the layer. Therefore, indicate that the ref counts must be + * loaded when the journal head has reaped past sequence number 1. + */ + update_slab_summary_entry(slab, &journal->slab_summary_waiter, + journal->summarized % journal->size, + (journal->head > 1), false, free_block_count); +} + +/** + * reopen_slab_journal() - Reopen a slab's journal by emptying it and then adding pending entries. + */ +static void reopen_slab_journal(struct vdo_slab *slab) +{ + struct slab_journal *journal = &slab->journal; + sequence_number_t block; + + VDO_ASSERT_LOG_ONLY(journal->tail_header.entry_count == 0, + "vdo_slab journal's active block empty before reopening"); + journal->head = journal->tail; + initialize_journal_state(journal); + + /* Ensure no locks are spuriously held on an empty journal. */ + for (block = 1; block <= journal->size; block++) { + VDO_ASSERT_LOG_ONLY((get_lock(journal, block)->count == 0), + "Scrubbed journal's block %llu is not locked", + (unsigned long long) block); + } + + add_entries(journal); +} + +static sequence_number_t get_committing_sequence_number(const struct pooled_vio *vio) +{ + const struct packed_slab_journal_block *block = + (const struct packed_slab_journal_block *) vio->vio.data; + + return __le64_to_cpu(block->header.sequence_number); +} + +/** + * complete_write() - Handle post-commit processing. + * @completion: The write vio as a completion. + * + * This is the callback registered by write_slab_journal_block(). + */ +static void complete_write(struct vdo_completion *completion) +{ + int result = completion->result; + struct pooled_vio *pooled = vio_as_pooled_vio(as_vio(completion)); + struct slab_journal *journal = completion->parent; + sequence_number_t committed = get_committing_sequence_number(pooled); + + list_del_init(&pooled->list_entry); + return_vio_to_pool(journal->slab->allocator->vio_pool, vdo_forget(pooled)); + + if (result != VDO_SUCCESS) { + vio_record_metadata_io_error(as_vio(completion)); + vdo_log_error_strerror(result, "cannot write slab journal block %llu", + (unsigned long long) committed); + vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo, result); + check_if_slab_drained(journal->slab); + return; + } + + WRITE_ONCE(journal->events->blocks_written, journal->events->blocks_written + 1); + + if (list_empty(&journal->uncommitted_blocks)) { + /* If no blocks are outstanding, then the commit point is at the tail. */ + journal->next_commit = journal->tail; + } else { + /* The commit point is always the beginning of the oldest incomplete block. */ + pooled = container_of(journal->uncommitted_blocks.next, + struct pooled_vio, list_entry); + journal->next_commit = get_committing_sequence_number(pooled); + } + + update_tail_block_location(journal); +} + +static void write_slab_journal_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct slab_journal *journal = vio->completion.parent; + + continue_vio_after_io(vio, complete_write, journal->slab->allocator->thread_id); +} + +/** + * write_slab_journal_block() - Write a slab journal block. + * @waiter: The vio pool waiter which was just notified. + * @context: The vio pool entry for the write. + * + * Callback from acquire_vio_from_pool() registered in commit_tail(). + */ +static void write_slab_journal_block(struct vdo_waiter *waiter, void *context) +{ + struct pooled_vio *pooled = context; + struct vio *vio = &pooled->vio; + struct slab_journal *journal = + container_of(waiter, struct slab_journal, resource_waiter); + struct slab_journal_block_header *header = &journal->tail_header; + int unused_entries = journal->entries_per_block - header->entry_count; + physical_block_number_t block_number; + const struct admin_state_code *operation; + + header->head = journal->head; + list_add_tail(&pooled->list_entry, &journal->uncommitted_blocks); + vdo_pack_slab_journal_block_header(header, &journal->block->header); + + /* Copy the tail block into the vio. */ + memcpy(pooled->vio.data, journal->block, VDO_BLOCK_SIZE); + + VDO_ASSERT_LOG_ONLY(unused_entries >= 0, "vdo_slab journal block is not overfull"); + if (unused_entries > 0) { + /* + * Release the per-entry locks for any unused entries in the block we are about to + * write. + */ + adjust_slab_journal_block_reference(journal, header->sequence_number, + -unused_entries); + journal->partial_write_in_progress = !block_is_full(journal); + } + + block_number = journal->slab->journal_origin + + (header->sequence_number % journal->size); + vio->completion.parent = journal; + + /* + * This block won't be read in recovery until the slab summary is updated to refer to it. + * The slab summary update does a flush which is sufficient to protect us from corruption + * due to out of order slab journal, reference block, or block map writes. + */ + vdo_submit_metadata_vio(vdo_forget(vio), block_number, write_slab_journal_endio, + complete_write, REQ_OP_WRITE); + + /* Since the write is submitted, the tail block structure can be reused. */ + journal->tail++; + initialize_tail_block(journal); + journal->waiting_to_commit = false; + + operation = vdo_get_admin_state_code(&journal->slab->state); + if (operation == VDO_ADMIN_STATE_WAITING_FOR_RECOVERY) { + vdo_finish_operation(&journal->slab->state, + (vdo_is_read_only(journal->slab->allocator->depot->vdo) ? + VDO_READ_ONLY : VDO_SUCCESS)); + return; + } + + add_entries(journal); +} + +/** + * commit_tail() - Commit the tail block of the slab journal. + * @journal: The journal whose tail block should be committed. + */ +static void commit_tail(struct slab_journal *journal) +{ + if ((journal->tail_header.entry_count == 0) && must_make_entries_to_flush(journal)) { + /* + * There are no entries at the moment, but there are some waiters, so defer + * initiating the flush until those entries are ready to write. + */ + return; + } + + if (vdo_is_read_only(journal->slab->allocator->depot->vdo) || + journal->waiting_to_commit || + (journal->tail_header.entry_count == 0)) { + /* + * There is nothing to do since the tail block is empty, or writing, or the journal + * is in read-only mode. + */ + return; + } + + /* + * Since we are about to commit the tail block, this journal no longer needs to be on the + * ring of journals which the recovery journal might ask to commit. + */ + mark_slab_journal_clean(journal); + + journal->waiting_to_commit = true; + + journal->resource_waiter.callback = write_slab_journal_block; + acquire_vio_from_pool(journal->slab->allocator->vio_pool, + &journal->resource_waiter); +} + +/** + * encode_slab_journal_entry() - Encode a slab journal entry. + * @tail_header: The unpacked header for the block. + * @payload: The journal block payload to hold the entry. + * @sbn: The slab block number of the entry to encode. + * @operation: The type of the entry. + * @increment: True if this is an increment. + * + * Exposed for unit tests. + */ +static void encode_slab_journal_entry(struct slab_journal_block_header *tail_header, + slab_journal_payload *payload, + slab_block_number sbn, + enum journal_operation operation, + bool increment) +{ + journal_entry_count_t entry_number = tail_header->entry_count++; + + if (operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) { + if (!tail_header->has_block_map_increments) { + memset(payload->full_entries.entry_types, 0, + VDO_SLAB_JOURNAL_ENTRY_TYPES_SIZE); + tail_header->has_block_map_increments = true; + } + + payload->full_entries.entry_types[entry_number / 8] |= + ((u8)1 << (entry_number % 8)); + } + + vdo_pack_slab_journal_entry(&payload->entries[entry_number], sbn, increment); +} + +/** + * expand_journal_point() - Convert a recovery journal journal_point which refers to both an + * increment and a decrement to a single point which refers to one or the + * other. + * @recovery_point: The journal point to convert. + * @increment: Whether the current entry is an increment. + * + * Return: The expanded journal point + * + * Because each data_vio has but a single recovery journal point, but may need to make both + * increment and decrement entries in the same slab journal. In order to distinguish the two + * entries, the entry count of the expanded journal point is twice the actual recovery journal + * entry count for increments, and one more than that for decrements. + */ +static struct journal_point expand_journal_point(struct journal_point recovery_point, + bool increment) +{ + recovery_point.entry_count *= 2; + if (!increment) + recovery_point.entry_count++; + + return recovery_point; +} + +/** + * add_entry() - Actually add an entry to the slab journal, potentially firing off a write if a + * block becomes full. + * @journal: The slab journal to append to. + * @pbn: The pbn being adjusted. + * @operation: The type of entry to make. + * @increment: True if this is an increment. + * @recovery_point: The expanded recovery point. + * + * This function is synchronous. + */ +static void add_entry(struct slab_journal *journal, physical_block_number_t pbn, + enum journal_operation operation, bool increment, + struct journal_point recovery_point) +{ + struct packed_slab_journal_block *block = journal->block; + int result; + + result = VDO_ASSERT(vdo_before_journal_point(&journal->tail_header.recovery_point, + &recovery_point), + "recovery journal point is monotonically increasing, recovery point: %llu.%u, block recovery point: %llu.%u", + (unsigned long long) recovery_point.sequence_number, + recovery_point.entry_count, + (unsigned long long) journal->tail_header.recovery_point.sequence_number, + journal->tail_header.recovery_point.entry_count); + if (result != VDO_SUCCESS) { + vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo, result); + return; + } + + if (operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) { + result = VDO_ASSERT((journal->tail_header.entry_count < + journal->full_entries_per_block), + "block has room for full entries"); + if (result != VDO_SUCCESS) { + vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo, + result); + return; + } + } + + encode_slab_journal_entry(&journal->tail_header, &block->payload, + pbn - journal->slab->start, operation, increment); + journal->tail_header.recovery_point = recovery_point; + if (block_is_full(journal)) + commit_tail(journal); +} + +static inline block_count_t journal_length(const struct slab_journal *journal) +{ + return journal->tail - journal->head; +} + +/** + * vdo_attempt_replay_into_slab() - Replay a recovery journal entry into a slab's journal. + * @slab: The slab to play into. + * @pbn: The PBN for the entry. + * @operation: The type of entry to add. + * @increment: True if this entry is an increment. + * @recovery_point: The recovery journal point corresponding to this entry. + * @parent: The completion to notify when there is space to add the entry if the entry could not be + * added immediately. + * + * Return: true if the entry was added immediately. + */ +bool vdo_attempt_replay_into_slab(struct vdo_slab *slab, physical_block_number_t pbn, + enum journal_operation operation, bool increment, + struct journal_point *recovery_point, + struct vdo_completion *parent) +{ + struct slab_journal *journal = &slab->journal; + struct slab_journal_block_header *header = &journal->tail_header; + struct journal_point expanded = expand_journal_point(*recovery_point, increment); + + /* Only accept entries after the current recovery point. */ + if (!vdo_before_journal_point(&journal->tail_header.recovery_point, &expanded)) + return true; + + if ((header->entry_count >= journal->full_entries_per_block) && + (header->has_block_map_increments || (operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING))) { + /* + * The tail block does not have room for the entry we are attempting to add so + * commit the tail block now. + */ + commit_tail(journal); + } + + if (journal->waiting_to_commit) { + vdo_start_operation_with_waiter(&journal->slab->state, + VDO_ADMIN_STATE_WAITING_FOR_RECOVERY, + parent, NULL); + return false; + } + + if (journal_length(journal) >= journal->size) { + /* + * We must have reaped the current head before the crash, since the blocked + * threshold keeps us from having more entries than fit in a slab journal; hence we + * can just advance the head (and unreapable block), as needed. + */ + journal->head++; + journal->unreapable++; + } + + if (journal->slab->status == VDO_SLAB_REBUILT) + journal->slab->status = VDO_SLAB_REPLAYING; + + add_entry(journal, pbn, operation, increment, expanded); + return true; +} + +/** + * requires_reaping() - Check whether the journal must be reaped before adding new entries. + * @journal: The journal to check. + * + * Return: true if the journal must be reaped. + */ +static bool requires_reaping(const struct slab_journal *journal) +{ + return (journal_length(journal) >= journal->blocking_threshold); +} + +/** finish_summary_update() - A waiter callback that resets the writing state of a slab. */ +static void finish_summary_update(struct vdo_waiter *waiter, void *context) +{ + struct vdo_slab *slab = container_of(waiter, struct vdo_slab, summary_waiter); + int result = *((int *) context); + + slab->active_count--; + + if ((result != VDO_SUCCESS) && (result != VDO_READ_ONLY)) { + vdo_log_error_strerror(result, "failed to update slab summary"); + vdo_enter_read_only_mode(slab->allocator->depot->vdo, result); + } + + check_if_slab_drained(slab); +} + +static void write_reference_block(struct vdo_waiter *waiter, void *context); + +/** + * launch_reference_block_write() - Launch the write of a dirty reference block by first acquiring + * a VIO for it from the pool. + * @waiter: The waiter of the block which is starting to write. + * @context: The parent slab of the block. + * + * This can be asynchronous since the writer will have to wait if all VIOs in the pool are + * currently in use. + */ +static void launch_reference_block_write(struct vdo_waiter *waiter, void *context) +{ + struct vdo_slab *slab = context; + + if (vdo_is_read_only(slab->allocator->depot->vdo)) + return; + + slab->active_count++; + container_of(waiter, struct reference_block, waiter)->is_writing = true; + waiter->callback = write_reference_block; + acquire_vio_from_pool(slab->allocator->vio_pool, waiter); +} + +static void save_dirty_reference_blocks(struct vdo_slab *slab) +{ + vdo_waitq_notify_all_waiters(&slab->dirty_blocks, + launch_reference_block_write, slab); + check_if_slab_drained(slab); +} + +/** + * finish_reference_block_write() - After a reference block has written, clean it, release its + * locks, and return its VIO to the pool. + * @completion: The VIO that just finished writing. + */ +static void finish_reference_block_write(struct vdo_completion *completion) +{ + struct vio *vio = as_vio(completion); + struct pooled_vio *pooled = vio_as_pooled_vio(vio); + struct reference_block *block = completion->parent; + struct vdo_slab *slab = block->slab; + tail_block_offset_t offset; + + slab->active_count--; + + /* Release the slab journal lock. */ + adjust_slab_journal_block_reference(&slab->journal, + block->slab_journal_lock_to_release, -1); + return_vio_to_pool(slab->allocator->vio_pool, pooled); + + /* + * We can't clear the is_writing flag earlier as releasing the slab journal lock may cause + * us to be dirtied again, but we don't want to double enqueue. + */ + block->is_writing = false; + + if (vdo_is_read_only(completion->vdo)) { + check_if_slab_drained(slab); + return; + } + + /* Re-queue the block if it was re-dirtied while it was writing. */ + if (block->is_dirty) { + vdo_waitq_enqueue_waiter(&block->slab->dirty_blocks, &block->waiter); + if (vdo_is_state_draining(&slab->state)) { + /* We must be saving, and this block will otherwise not be relaunched. */ + save_dirty_reference_blocks(slab); + } + + return; + } + + /* + * Mark the slab as clean in the slab summary if there are no dirty or writing blocks + * and no summary update in progress. + */ + if ((slab->active_count > 0) || vdo_waitq_has_waiters(&slab->dirty_blocks)) { + check_if_slab_drained(slab); + return; + } + + offset = slab->allocator->summary_entries[slab->slab_number].tail_block_offset; + slab->active_count++; + slab->summary_waiter.callback = finish_summary_update; + update_slab_summary_entry(slab, &slab->summary_waiter, offset, + true, true, slab->free_blocks); +} + +/** + * get_reference_counters_for_block() - Find the reference counters for a given block. + * @block: The reference_block in question. + * + * Return: A pointer to the reference counters for this block. + */ +static vdo_refcount_t * __must_check get_reference_counters_for_block(struct reference_block *block) +{ + size_t block_index = block - block->slab->reference_blocks; + + return &block->slab->counters[block_index * COUNTS_PER_BLOCK]; +} + +/** + * pack_reference_block() - Copy data from a reference block to a buffer ready to be written out. + * @block: The block to copy. + * @buffer: The char buffer to fill with the packed block. + */ +static void pack_reference_block(struct reference_block *block, void *buffer) +{ + struct packed_reference_block *packed = buffer; + vdo_refcount_t *counters = get_reference_counters_for_block(block); + sector_count_t i; + struct packed_journal_point commit_point; + + vdo_pack_journal_point(&block->slab->slab_journal_point, &commit_point); + + for (i = 0; i < VDO_SECTORS_PER_BLOCK; i++) { + packed->sectors[i].commit_point = commit_point; + memcpy(packed->sectors[i].counts, counters + (i * COUNTS_PER_SECTOR), + (sizeof(vdo_refcount_t) * COUNTS_PER_SECTOR)); + } +} + +static void write_reference_block_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct reference_block *block = vio->completion.parent; + thread_id_t thread_id = block->slab->allocator->thread_id; + + continue_vio_after_io(vio, finish_reference_block_write, thread_id); +} + +/** + * handle_io_error() - Handle an I/O error reading or writing a reference count block. + * @completion: The VIO doing the I/O as a completion. + */ +static void handle_io_error(struct vdo_completion *completion) +{ + int result = completion->result; + struct vio *vio = as_vio(completion); + struct vdo_slab *slab = ((struct reference_block *) completion->parent)->slab; + + vio_record_metadata_io_error(vio); + return_vio_to_pool(slab->allocator->vio_pool, vio_as_pooled_vio(vio)); + slab->active_count--; + vdo_enter_read_only_mode(slab->allocator->depot->vdo, result); + check_if_slab_drained(slab); +} + +/** + * write_reference_block() - After a dirty block waiter has gotten a VIO from the VIO pool, copy + * its counters and associated data into the VIO, and launch the write. + * @waiter: The waiter of the dirty block. + * @context: The VIO returned by the pool. + */ +static void write_reference_block(struct vdo_waiter *waiter, void *context) +{ + size_t block_offset; + physical_block_number_t pbn; + struct pooled_vio *pooled = context; + struct vdo_completion *completion = &pooled->vio.completion; + struct reference_block *block = container_of(waiter, struct reference_block, + waiter); + + pack_reference_block(block, pooled->vio.data); + block_offset = (block - block->slab->reference_blocks); + pbn = (block->slab->ref_counts_origin + block_offset); + block->slab_journal_lock_to_release = block->slab_journal_lock; + completion->parent = block; + + /* + * Mark the block as clean, since we won't be committing any updates that happen after this + * moment. As long as VIO order is preserved, two VIOs updating this block at once will not + * cause complications. + */ + block->is_dirty = false; + + /* + * Flush before writing to ensure that the recovery journal and slab journal entries which + * cover this reference update are stable. This prevents data corruption that can be caused + * by out of order writes. + */ + WRITE_ONCE(block->slab->allocator->ref_counts_statistics.blocks_written, + block->slab->allocator->ref_counts_statistics.blocks_written + 1); + + completion->callback_thread_id = ((struct block_allocator *) pooled->context)->thread_id; + vdo_submit_metadata_vio(&pooled->vio, pbn, write_reference_block_endio, + handle_io_error, REQ_OP_WRITE | REQ_PREFLUSH); +} + +static void reclaim_journal_space(struct slab_journal *journal) +{ + block_count_t length = journal_length(journal); + struct vdo_slab *slab = journal->slab; + block_count_t write_count = vdo_waitq_num_waiters(&slab->dirty_blocks); + block_count_t written; + + if ((length < journal->flushing_threshold) || (write_count == 0)) + return; + + /* The slab journal is over the first threshold, schedule some reference block writes. */ + WRITE_ONCE(journal->events->flush_count, journal->events->flush_count + 1); + if (length < journal->flushing_deadline) { + /* Schedule more writes the closer to the deadline we get. */ + write_count /= journal->flushing_deadline - length + 1; + write_count = max_t(block_count_t, write_count, 1); + } + + for (written = 0; written < write_count; written++) { + vdo_waitq_notify_next_waiter(&slab->dirty_blocks, + launch_reference_block_write, slab); + } +} + +/** + * reference_count_to_status() - Convert a reference count to a reference status. + * @count: The count to convert. + * + * Return: The appropriate reference status. + */ +static enum reference_status __must_check reference_count_to_status(vdo_refcount_t count) +{ + if (count == EMPTY_REFERENCE_COUNT) + return RS_FREE; + else if (count == 1) + return RS_SINGLE; + else if (count == PROVISIONAL_REFERENCE_COUNT) + return RS_PROVISIONAL; + else + return RS_SHARED; +} + +/** + * dirty_block() - Mark a reference count block as dirty, potentially adding it to the dirty queue + * if it wasn't already dirty. + * @block: The reference block to mark as dirty. + */ +static void dirty_block(struct reference_block *block) +{ + if (block->is_dirty) + return; + + block->is_dirty = true; + if (!block->is_writing) + vdo_waitq_enqueue_waiter(&block->slab->dirty_blocks, &block->waiter); +} + +/** + * get_reference_block() - Get the reference block that covers the given block index. + */ +static struct reference_block * __must_check get_reference_block(struct vdo_slab *slab, + slab_block_number index) +{ + return &slab->reference_blocks[index / COUNTS_PER_BLOCK]; +} + +/** + * slab_block_number_from_pbn() - Determine the index within the slab of a particular physical + * block number. + * @slab: The slab. + * @physical_block_number: The physical block number. + * @slab_block_number_ptr: A pointer to the slab block number. + * + * Return: VDO_SUCCESS or an error code. + */ +static int __must_check slab_block_number_from_pbn(struct vdo_slab *slab, + physical_block_number_t pbn, + slab_block_number *slab_block_number_ptr) +{ + u64 slab_block_number; + + if (pbn < slab->start) + return VDO_OUT_OF_RANGE; + + slab_block_number = pbn - slab->start; + if (slab_block_number >= slab->allocator->depot->slab_config.data_blocks) + return VDO_OUT_OF_RANGE; + + *slab_block_number_ptr = slab_block_number; + return VDO_SUCCESS; +} + +/** + * get_reference_counter() - Get the reference counter that covers the given physical block number. + * @slab: The slab to query. + * @pbn: The physical block number. + * @counter_ptr: A pointer to the reference counter. + */ +static int __must_check get_reference_counter(struct vdo_slab *slab, + physical_block_number_t pbn, + vdo_refcount_t **counter_ptr) +{ + slab_block_number index; + int result = slab_block_number_from_pbn(slab, pbn, &index); + + if (result != VDO_SUCCESS) + return result; + + *counter_ptr = &slab->counters[index]; + + return VDO_SUCCESS; +} + +static unsigned int calculate_slab_priority(struct vdo_slab *slab) +{ + block_count_t free_blocks = slab->free_blocks; + unsigned int unopened_slab_priority = slab->allocator->unopened_slab_priority; + unsigned int priority; + + /* + * Wholly full slabs must be the only ones with lowest priority, 0. + * + * Slabs that have never been opened (empty, newly initialized, and never been written to) + * have lower priority than previously opened slabs that have a significant number of free + * blocks. This ranking causes VDO to avoid writing physical blocks for the first time + * unless there are very few free blocks that have been previously written to. + * + * Since VDO doesn't discard blocks currently, reusing previously written blocks makes VDO + * a better client of any underlying storage that is thinly-provisioned (though discarding + * would be better). + * + * For all other slabs, the priority is derived from the logarithm of the number of free + * blocks. Slabs with the same order of magnitude of free blocks have the same priority. + * With 2^23 blocks, the priority will range from 1 to 25. The reserved + * unopened_slab_priority divides the range and is skipped by the logarithmic mapping. + */ + + if (free_blocks == 0) + return 0; + + if (is_slab_journal_blank(slab)) + return unopened_slab_priority; + + priority = (1 + ilog2(free_blocks)); + return ((priority < unopened_slab_priority) ? priority : priority + 1); +} + +/* + * Slabs are essentially prioritized by an approximation of the number of free blocks in the slab + * so slabs with lots of free blocks will be opened for allocation before slabs that have few free + * blocks. + */ +static void prioritize_slab(struct vdo_slab *slab) +{ + VDO_ASSERT_LOG_ONLY(list_empty(&slab->allocq_entry), + "a slab must not already be on a ring when prioritizing"); + slab->priority = calculate_slab_priority(slab); + vdo_priority_table_enqueue(slab->allocator->prioritized_slabs, + slab->priority, &slab->allocq_entry); +} + +/** + * adjust_free_block_count() - Adjust the free block count and (if needed) reprioritize the slab. + * @incremented: true if the free block count went up. + */ +static void adjust_free_block_count(struct vdo_slab *slab, bool incremented) +{ + struct block_allocator *allocator = slab->allocator; + + WRITE_ONCE(allocator->allocated_blocks, + allocator->allocated_blocks + (incremented ? -1 : 1)); + + /* The open slab doesn't need to be reprioritized until it is closed. */ + if (slab == allocator->open_slab) + return; + + /* Don't bother adjusting the priority table if unneeded. */ + if (slab->priority == calculate_slab_priority(slab)) + return; + + /* + * Reprioritize the slab to reflect the new free block count by removing it from the table + * and re-enqueuing it with the new priority. + */ + vdo_priority_table_remove(allocator->prioritized_slabs, &slab->allocq_entry); + prioritize_slab(slab); +} + +/** + * increment_for_data() - Increment the reference count for a data block. + * @slab: The slab which owns the block. + * @block: The reference block which contains the block being updated. + * @block_number: The block to update. + * @old_status: The reference status of the data block before this increment. + * @lock: The pbn_lock associated with this increment (may be NULL). + * @counter_ptr: A pointer to the count for the data block (in, out). + * @adjust_block_count: Whether to update the allocator's free block count. + * + * Return: VDO_SUCCESS or an error. + */ +static int increment_for_data(struct vdo_slab *slab, struct reference_block *block, + slab_block_number block_number, + enum reference_status old_status, + struct pbn_lock *lock, vdo_refcount_t *counter_ptr, + bool adjust_block_count) +{ + switch (old_status) { + case RS_FREE: + *counter_ptr = 1; + block->allocated_count++; + slab->free_blocks--; + if (adjust_block_count) + adjust_free_block_count(slab, false); + + break; + + case RS_PROVISIONAL: + *counter_ptr = 1; + break; + + default: + /* Single or shared */ + if (*counter_ptr >= MAXIMUM_REFERENCE_COUNT) { + return vdo_log_error_strerror(VDO_REF_COUNT_INVALID, + "Incrementing a block already having 254 references (slab %u, offset %u)", + slab->slab_number, block_number); + } + (*counter_ptr)++; + } + + if (lock != NULL) + vdo_unassign_pbn_lock_provisional_reference(lock); + return VDO_SUCCESS; +} + +/** + * decrement_for_data() - Decrement the reference count for a data block. + * @slab: The slab which owns the block. + * @block: The reference block which contains the block being updated. + * @block_number: The block to update. + * @old_status: The reference status of the data block before this decrement. + * @updater: The reference updater doing this operation in case we need to look up the pbn lock. + * @lock: The pbn_lock associated with the block being decremented (may be NULL). + * @counter_ptr: A pointer to the count for the data block (in, out). + * @adjust_block_count: Whether to update the allocator's free block count. + * + * Return: VDO_SUCCESS or an error. + */ +static int decrement_for_data(struct vdo_slab *slab, struct reference_block *block, + slab_block_number block_number, + enum reference_status old_status, + struct reference_updater *updater, + vdo_refcount_t *counter_ptr, bool adjust_block_count) +{ + switch (old_status) { + case RS_FREE: + return vdo_log_error_strerror(VDO_REF_COUNT_INVALID, + "Decrementing free block at offset %u in slab %u", + block_number, slab->slab_number); + + case RS_PROVISIONAL: + case RS_SINGLE: + if (updater->zpbn.zone != NULL) { + struct pbn_lock *lock = vdo_get_physical_zone_pbn_lock(updater->zpbn.zone, + updater->zpbn.pbn); + + if (lock != NULL) { + /* + * There is a read lock on this block, so the block must not become + * unreferenced. + */ + *counter_ptr = PROVISIONAL_REFERENCE_COUNT; + vdo_assign_pbn_lock_provisional_reference(lock); + break; + } + } + + *counter_ptr = EMPTY_REFERENCE_COUNT; + block->allocated_count--; + slab->free_blocks++; + if (adjust_block_count) + adjust_free_block_count(slab, true); + + break; + + default: + /* Shared */ + (*counter_ptr)--; + } + + return VDO_SUCCESS; +} + +/** + * increment_for_block_map() - Increment the reference count for a block map page. + * @slab: The slab which owns the block. + * @block: The reference block which contains the block being updated. + * @block_number: The block to update. + * @old_status: The reference status of the block before this increment. + * @lock: The pbn_lock associated with this increment (may be NULL). + * @normal_operation: Whether we are in normal operation vs. recovery or rebuild. + * @counter_ptr: A pointer to the count for the block (in, out). + * @adjust_block_count: Whether to update the allocator's free block count. + * + * All block map increments should be from provisional to MAXIMUM_REFERENCE_COUNT. Since block map + * blocks never dedupe they should never be adjusted from any other state. The adjustment always + * results in MAXIMUM_REFERENCE_COUNT as this value is used to prevent dedupe against block map + * blocks. + * + * Return: VDO_SUCCESS or an error. + */ +static int increment_for_block_map(struct vdo_slab *slab, struct reference_block *block, + slab_block_number block_number, + enum reference_status old_status, + struct pbn_lock *lock, bool normal_operation, + vdo_refcount_t *counter_ptr, bool adjust_block_count) +{ + switch (old_status) { + case RS_FREE: + if (normal_operation) { + return vdo_log_error_strerror(VDO_REF_COUNT_INVALID, + "Incrementing unallocated block map block (slab %u, offset %u)", + slab->slab_number, block_number); + } + + *counter_ptr = MAXIMUM_REFERENCE_COUNT; + block->allocated_count++; + slab->free_blocks--; + if (adjust_block_count) + adjust_free_block_count(slab, false); + + return VDO_SUCCESS; + + case RS_PROVISIONAL: + if (!normal_operation) + return vdo_log_error_strerror(VDO_REF_COUNT_INVALID, + "Block map block had provisional reference during replay (slab %u, offset %u)", + slab->slab_number, block_number); + + *counter_ptr = MAXIMUM_REFERENCE_COUNT; + if (lock != NULL) + vdo_unassign_pbn_lock_provisional_reference(lock); + return VDO_SUCCESS; + + default: + return vdo_log_error_strerror(VDO_REF_COUNT_INVALID, + "Incrementing a block map block which is already referenced %u times (slab %u, offset %u)", + *counter_ptr, slab->slab_number, + block_number); + } +} + +static bool __must_check is_valid_journal_point(const struct journal_point *point) +{ + return ((point != NULL) && (point->sequence_number > 0)); +} + +/** + * update_reference_count() - Update the reference count of a block. + * @slab: The slab which owns the block. + * @block: The reference block which contains the block being updated. + * @block_number: The block to update. + * @slab_journal_point: The slab journal point at which this update is journaled. + * @updater: The reference updater. + * @normal_operation: Whether we are in normal operation vs. recovery or rebuild. + * @adjust_block_count: Whether to update the slab's free block count. + * @provisional_decrement_ptr: A pointer which will be set to true if this update was a decrement + * of a provisional reference. + * + * Return: VDO_SUCCESS or an error. + */ +static int update_reference_count(struct vdo_slab *slab, struct reference_block *block, + slab_block_number block_number, + const struct journal_point *slab_journal_point, + struct reference_updater *updater, + bool normal_operation, bool adjust_block_count, + bool *provisional_decrement_ptr) +{ + vdo_refcount_t *counter_ptr = &slab->counters[block_number]; + enum reference_status old_status = reference_count_to_status(*counter_ptr); + int result; + + if (!updater->increment) { + result = decrement_for_data(slab, block, block_number, old_status, + updater, counter_ptr, adjust_block_count); + if ((result == VDO_SUCCESS) && (old_status == RS_PROVISIONAL)) { + if (provisional_decrement_ptr != NULL) + *provisional_decrement_ptr = true; + return VDO_SUCCESS; + } + } else if (updater->operation == VDO_JOURNAL_DATA_REMAPPING) { + result = increment_for_data(slab, block, block_number, old_status, + updater->lock, counter_ptr, adjust_block_count); + } else { + result = increment_for_block_map(slab, block, block_number, old_status, + updater->lock, normal_operation, + counter_ptr, adjust_block_count); + } + + if (result != VDO_SUCCESS) + return result; + + if (is_valid_journal_point(slab_journal_point)) + slab->slab_journal_point = *slab_journal_point; + + return VDO_SUCCESS; +} + +static int __must_check adjust_reference_count(struct vdo_slab *slab, + struct reference_updater *updater, + const struct journal_point *slab_journal_point) +{ + slab_block_number block_number; + int result; + struct reference_block *block; + bool provisional_decrement = false; + + if (!is_slab_open(slab)) + return VDO_INVALID_ADMIN_STATE; + + result = slab_block_number_from_pbn(slab, updater->zpbn.pbn, &block_number); + if (result != VDO_SUCCESS) + return result; + + block = get_reference_block(slab, block_number); + result = update_reference_count(slab, block, block_number, slab_journal_point, + updater, NORMAL_OPERATION, true, + &provisional_decrement); + if ((result != VDO_SUCCESS) || provisional_decrement) + return result; + + if (block->is_dirty && (block->slab_journal_lock > 0)) { + sequence_number_t entry_lock = slab_journal_point->sequence_number; + /* + * This block is already dirty and a slab journal entry has been made for it since + * the last time it was clean. We must release the per-entry slab journal lock for + * the entry associated with the update we are now doing. + */ + result = VDO_ASSERT(is_valid_journal_point(slab_journal_point), + "Reference count adjustments need slab journal points."); + if (result != VDO_SUCCESS) + return result; + + adjust_slab_journal_block_reference(&slab->journal, entry_lock, -1); + return VDO_SUCCESS; + } + + /* + * This may be the first time we are applying an update for which there is a slab journal + * entry to this block since the block was cleaned. Therefore, we convert the per-entry + * slab journal lock to an uncommitted reference block lock, if there is a per-entry lock. + */ + if (is_valid_journal_point(slab_journal_point)) + block->slab_journal_lock = slab_journal_point->sequence_number; + else + block->slab_journal_lock = 0; + + dirty_block(block); + return VDO_SUCCESS; +} + +/** + * add_entry_from_waiter() - Add an entry to the slab journal. + * @waiter: The vio which should make an entry now. + * @context: The slab journal to make an entry in. + * + * This callback is invoked by add_entries() once it has determined that we are ready to make + * another entry in the slab journal. Implements waiter_callback_fn. + */ +static void add_entry_from_waiter(struct vdo_waiter *waiter, void *context) +{ + int result; + struct reference_updater *updater = + container_of(waiter, struct reference_updater, waiter); + struct data_vio *data_vio = data_vio_from_reference_updater(updater); + struct slab_journal *journal = context; + struct slab_journal_block_header *header = &journal->tail_header; + struct journal_point slab_journal_point = { + .sequence_number = header->sequence_number, + .entry_count = header->entry_count, + }; + sequence_number_t recovery_block = data_vio->recovery_journal_point.sequence_number; + + if (header->entry_count == 0) { + /* + * This is the first entry in the current tail block, so get a lock on the recovery + * journal which we will hold until this tail block is committed. + */ + get_lock(journal, header->sequence_number)->recovery_start = recovery_block; + if (journal->recovery_journal != NULL) { + zone_count_t zone_number = journal->slab->allocator->zone_number; + + vdo_acquire_recovery_journal_block_reference(journal->recovery_journal, + recovery_block, + VDO_ZONE_TYPE_PHYSICAL, + zone_number); + } + + mark_slab_journal_dirty(journal, recovery_block); + reclaim_journal_space(journal); + } + + add_entry(journal, updater->zpbn.pbn, updater->operation, updater->increment, + expand_journal_point(data_vio->recovery_journal_point, + updater->increment)); + + if (journal->slab->status != VDO_SLAB_REBUILT) { + /* + * If the slab is unrecovered, scrubbing will take care of the count since the + * update is now recorded in the journal. + */ + adjust_slab_journal_block_reference(journal, + slab_journal_point.sequence_number, -1); + result = VDO_SUCCESS; + } else { + /* Now that an entry has been made in the slab journal, update the counter. */ + result = adjust_reference_count(journal->slab, updater, + &slab_journal_point); + } + + if (updater->increment) + continue_data_vio_with_error(data_vio, result); + else + vdo_continue_completion(&data_vio->decrement_completion, result); +} + +/** + * is_next_entry_a_block_map_increment() - Check whether the next entry to be made is a block map + * increment. + * @journal: The journal. + * + * Return: true if the first entry waiter's operation is a block map increment. + */ +static inline bool is_next_entry_a_block_map_increment(struct slab_journal *journal) +{ + struct vdo_waiter *waiter = vdo_waitq_get_first_waiter(&journal->entry_waiters); + struct reference_updater *updater = + container_of(waiter, struct reference_updater, waiter); + + return (updater->operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING); +} + +/** + * add_entries() - Add as many entries as possible from the queue of vios waiting to make entries. + * @journal: The journal to which entries may be added. + * + * By processing the queue in order, we ensure that slab journal entries are made in the same order + * as recovery journal entries for the same increment or decrement. + */ +static void add_entries(struct slab_journal *journal) +{ + if (journal->adding_entries) { + /* Protect against re-entrancy. */ + return; + } + + journal->adding_entries = true; + while (vdo_waitq_has_waiters(&journal->entry_waiters)) { + struct slab_journal_block_header *header = &journal->tail_header; + + if (journal->partial_write_in_progress || + (journal->slab->status == VDO_SLAB_REBUILDING)) { + /* + * Don't add entries while rebuilding or while a partial write is + * outstanding, as it could result in reference count corruption. + */ + break; + } + + if (journal->waiting_to_commit) { + /* + * If we are waiting for resources to write the tail block, and the tail + * block is full, we can't make another entry. + */ + WRITE_ONCE(journal->events->tail_busy_count, + journal->events->tail_busy_count + 1); + break; + } else if (is_next_entry_a_block_map_increment(journal) && + (header->entry_count >= journal->full_entries_per_block)) { + /* + * The tail block does not have room for a block map increment, so commit + * it now. + */ + commit_tail(journal); + if (journal->waiting_to_commit) { + WRITE_ONCE(journal->events->tail_busy_count, + journal->events->tail_busy_count + 1); + break; + } + } + + /* If the slab is over the blocking threshold, make the vio wait. */ + if (requires_reaping(journal)) { + WRITE_ONCE(journal->events->blocked_count, + journal->events->blocked_count + 1); + save_dirty_reference_blocks(journal->slab); + break; + } + + if (header->entry_count == 0) { + struct journal_lock *lock = + get_lock(journal, header->sequence_number); + + /* + * Check if the on disk slab journal is full. Because of the blocking and + * scrubbing thresholds, this should never happen. + */ + if (lock->count > 0) { + VDO_ASSERT_LOG_ONLY((journal->head + journal->size) == journal->tail, + "New block has locks, but journal is not full"); + + /* + * The blocking threshold must let the journal fill up if the new + * block has locks; if the blocking threshold is smaller than the + * journal size, the new block cannot possibly have locks already. + */ + VDO_ASSERT_LOG_ONLY((journal->blocking_threshold >= journal->size), + "New block can have locks already iff blocking threshold is at the end of the journal"); + + WRITE_ONCE(journal->events->disk_full_count, + journal->events->disk_full_count + 1); + save_dirty_reference_blocks(journal->slab); + break; + } + + /* + * Don't allow the new block to be reaped until all of the reference count + * blocks are written and the journal block has been fully committed as + * well. + */ + lock->count = journal->entries_per_block + 1; + + if (header->sequence_number == 1) { + struct vdo_slab *slab = journal->slab; + block_count_t i; + + /* + * This is the first entry in this slab journal, ever. Dirty all of + * the reference count blocks. Each will acquire a lock on the tail + * block so that the journal won't be reaped until the reference + * counts are initialized. The lock acquisition must be done by the + * ref_counts since here we don't know how many reference blocks + * the ref_counts has. + */ + for (i = 0; i < slab->reference_block_count; i++) { + slab->reference_blocks[i].slab_journal_lock = 1; + dirty_block(&slab->reference_blocks[i]); + } + + adjust_slab_journal_block_reference(journal, 1, + slab->reference_block_count); + } + } + + vdo_waitq_notify_next_waiter(&journal->entry_waiters, + add_entry_from_waiter, journal); + } + + journal->adding_entries = false; + + /* If there are no waiters, and we are flushing or saving, commit the tail block. */ + if (vdo_is_state_draining(&journal->slab->state) && + !vdo_is_state_suspending(&journal->slab->state) && + !vdo_waitq_has_waiters(&journal->entry_waiters)) + commit_tail(journal); +} + +/** + * reset_search_cursor() - Reset the free block search back to the first reference counter in the + * first reference block of a slab. + */ +static void reset_search_cursor(struct vdo_slab *slab) +{ + struct search_cursor *cursor = &slab->search_cursor; + + cursor->block = cursor->first_block; + cursor->index = 0; + /* Unit tests have slabs with only one reference block (and it's a runt). */ + cursor->end_index = min_t(u32, COUNTS_PER_BLOCK, slab->block_count); +} + +/** + * advance_search_cursor() - Advance the search cursor to the start of the next reference block in + * a slab, + * + * Wraps around to the first reference block if the current block is the last reference block. + * + * Return: true unless the cursor was at the last reference block. + */ +static bool advance_search_cursor(struct vdo_slab *slab) +{ + struct search_cursor *cursor = &slab->search_cursor; + + /* + * If we just finished searching the last reference block, then wrap back around to the + * start of the array. + */ + if (cursor->block == cursor->last_block) { + reset_search_cursor(slab); + return false; + } + + /* We're not already at the end, so advance to cursor to the next block. */ + cursor->block++; + cursor->index = cursor->end_index; + + if (cursor->block == cursor->last_block) { + /* The last reference block will usually be a runt. */ + cursor->end_index = slab->block_count; + } else { + cursor->end_index += COUNTS_PER_BLOCK; + } + + return true; +} + +/** + * vdo_adjust_reference_count_for_rebuild() - Adjust the reference count of a block during rebuild. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_adjust_reference_count_for_rebuild(struct slab_depot *depot, + physical_block_number_t pbn, + enum journal_operation operation) +{ + int result; + slab_block_number block_number; + struct reference_block *block; + struct vdo_slab *slab = vdo_get_slab(depot, pbn); + struct reference_updater updater = { + .operation = operation, + .increment = true, + }; + + result = slab_block_number_from_pbn(slab, pbn, &block_number); + if (result != VDO_SUCCESS) + return result; + + block = get_reference_block(slab, block_number); + result = update_reference_count(slab, block, block_number, NULL, + &updater, !NORMAL_OPERATION, false, NULL); + if (result != VDO_SUCCESS) + return result; + + dirty_block(block); + return VDO_SUCCESS; +} + +/** + * replay_reference_count_change() - Replay the reference count adjustment from a slab journal + * entry into the reference count for a block. + * @slab: The slab. + * @entry_point: The slab journal point for the entry. + * @entry: The slab journal entry being replayed. + * + * The adjustment will be ignored if it was already recorded in the reference count. + * + * Return: VDO_SUCCESS or an error code. + */ +static int replay_reference_count_change(struct vdo_slab *slab, + const struct journal_point *entry_point, + struct slab_journal_entry entry) +{ + int result; + struct reference_block *block = get_reference_block(slab, entry.sbn); + sector_count_t sector = (entry.sbn % COUNTS_PER_BLOCK) / COUNTS_PER_SECTOR; + struct reference_updater updater = { + .operation = entry.operation, + .increment = entry.increment, + }; + + if (!vdo_before_journal_point(&block->commit_points[sector], entry_point)) { + /* This entry is already reflected in the existing counts, so do nothing. */ + return VDO_SUCCESS; + } + + /* This entry is not yet counted in the reference counts. */ + result = update_reference_count(slab, block, entry.sbn, entry_point, + &updater, !NORMAL_OPERATION, false, NULL); + if (result != VDO_SUCCESS) + return result; + + dirty_block(block); + return VDO_SUCCESS; +} + +/** + * find_zero_byte_in_word() - Find the array index of the first zero byte in word-sized range of + * reference counters. + * @word_ptr: A pointer to the eight counter bytes to check. + * @start_index: The array index corresponding to word_ptr[0]. + * @fail_index: The array index to return if no zero byte is found. + * + * The search does no bounds checking; the function relies on the array being sufficiently padded. + * + * Return: The array index of the first zero byte in the word, or the value passed as fail_index if + * no zero byte was found. + */ +static inline slab_block_number find_zero_byte_in_word(const u8 *word_ptr, + slab_block_number start_index, + slab_block_number fail_index) +{ + u64 word = get_unaligned_le64(word_ptr); + + /* This looks like a loop, but GCC will unroll the eight iterations for us. */ + unsigned int offset; + + for (offset = 0; offset < BYTES_PER_WORD; offset++) { + /* Assumes little-endian byte order, which we have on X86. */ + if ((word & 0xFF) == 0) + return (start_index + offset); + word >>= 8; + } + + return fail_index; +} + +/** + * find_free_block() - Find the first block with a reference count of zero in the specified + * range of reference counter indexes. + * @slab: The slab counters to scan. + * @index_ptr: A pointer to hold the array index of the free block. + * + * Exposed for unit testing. + * + * Return: true if a free block was found in the specified range. + */ +static bool find_free_block(const struct vdo_slab *slab, slab_block_number *index_ptr) +{ + slab_block_number zero_index; + slab_block_number next_index = slab->search_cursor.index; + slab_block_number end_index = slab->search_cursor.end_index; + u8 *next_counter = &slab->counters[next_index]; + u8 *end_counter = &slab->counters[end_index]; + + /* + * Search every byte of the first unaligned word. (Array is padded so reading past end is + * safe.) + */ + zero_index = find_zero_byte_in_word(next_counter, next_index, end_index); + if (zero_index < end_index) { + *index_ptr = zero_index; + return true; + } + + /* + * On architectures where unaligned word access is expensive, this would be a good place to + * advance to an alignment boundary. + */ + next_index += BYTES_PER_WORD; + next_counter += BYTES_PER_WORD; + + /* + * Now we're word-aligned; check an word at a time until we find a word containing a zero. + * (Array is padded so reading past end is safe.) + */ + while (next_counter < end_counter) { + /* + * The following code is currently an exact copy of the code preceding the loop, + * but if you try to merge them by using a do loop, it runs slower because a jump + * instruction gets added at the start of the iteration. + */ + zero_index = find_zero_byte_in_word(next_counter, next_index, end_index); + if (zero_index < end_index) { + *index_ptr = zero_index; + return true; + } + + next_index += BYTES_PER_WORD; + next_counter += BYTES_PER_WORD; + } + + return false; +} + +/** + * search_current_reference_block() - Search the reference block currently saved in the search + * cursor for a reference count of zero, starting at the saved + * counter index. + * @slab: The slab to search. + * @free_index_ptr: A pointer to receive the array index of the zero reference count. + * + * Return: true if an unreferenced counter was found. + */ +static bool search_current_reference_block(const struct vdo_slab *slab, + slab_block_number *free_index_ptr) +{ + /* Don't bother searching if the current block is known to be full. */ + return ((slab->search_cursor.block->allocated_count < COUNTS_PER_BLOCK) && + find_free_block(slab, free_index_ptr)); +} + +/** + * search_reference_blocks() - Search each reference block for a reference count of zero. + * @slab: The slab to search. + * @free_index_ptr: A pointer to receive the array index of the zero reference count. + * + * Searches each reference block for a reference count of zero, starting at the reference block and + * counter index saved in the search cursor and searching up to the end of the last reference + * block. The search does not wrap. + * + * Return: true if an unreferenced counter was found. + */ +static bool search_reference_blocks(struct vdo_slab *slab, + slab_block_number *free_index_ptr) +{ + /* Start searching at the saved search position in the current block. */ + if (search_current_reference_block(slab, free_index_ptr)) + return true; + + /* Search each reference block up to the end of the slab. */ + while (advance_search_cursor(slab)) { + if (search_current_reference_block(slab, free_index_ptr)) + return true; + } + + return false; +} + +/** + * make_provisional_reference() - Do the bookkeeping for making a provisional reference. + */ +static void make_provisional_reference(struct vdo_slab *slab, + slab_block_number block_number) +{ + struct reference_block *block = get_reference_block(slab, block_number); + + /* + * Make the initial transition from an unreferenced block to a + * provisionally allocated block. + */ + slab->counters[block_number] = PROVISIONAL_REFERENCE_COUNT; + + /* Account for the allocation. */ + block->allocated_count++; + slab->free_blocks--; +} + +/** + * dirty_all_reference_blocks() - Mark all reference count blocks in a slab as dirty. + */ +static void dirty_all_reference_blocks(struct vdo_slab *slab) +{ + block_count_t i; + + for (i = 0; i < slab->reference_block_count; i++) + dirty_block(&slab->reference_blocks[i]); +} + +/** + * clear_provisional_references() - Clear the provisional reference counts from a reference block. + * @block: The block to clear. + */ +static void clear_provisional_references(struct reference_block *block) +{ + vdo_refcount_t *counters = get_reference_counters_for_block(block); + block_count_t j; + + for (j = 0; j < COUNTS_PER_BLOCK; j++) { + if (counters[j] == PROVISIONAL_REFERENCE_COUNT) { + counters[j] = EMPTY_REFERENCE_COUNT; + block->allocated_count--; + } + } +} + +static inline bool journal_points_equal(struct journal_point first, + struct journal_point second) +{ + return ((first.sequence_number == second.sequence_number) && + (first.entry_count == second.entry_count)); +} + +/** + * unpack_reference_block() - Unpack reference counts blocks into the internal memory structure. + * @packed: The written reference block to be unpacked. + * @block: The internal reference block to be loaded. + */ +static void unpack_reference_block(struct packed_reference_block *packed, + struct reference_block *block) +{ + block_count_t index; + sector_count_t i; + struct vdo_slab *slab = block->slab; + vdo_refcount_t *counters = get_reference_counters_for_block(block); + + for (i = 0; i < VDO_SECTORS_PER_BLOCK; i++) { + struct packed_reference_sector *sector = &packed->sectors[i]; + + vdo_unpack_journal_point(§or->commit_point, &block->commit_points[i]); + memcpy(counters + (i * COUNTS_PER_SECTOR), sector->counts, + (sizeof(vdo_refcount_t) * COUNTS_PER_SECTOR)); + /* The slab_journal_point must be the latest point found in any sector. */ + if (vdo_before_journal_point(&slab->slab_journal_point, + &block->commit_points[i])) + slab->slab_journal_point = block->commit_points[i]; + + if ((i > 0) && + !journal_points_equal(block->commit_points[0], + block->commit_points[i])) { + size_t block_index = block - block->slab->reference_blocks; + + vdo_log_warning("Torn write detected in sector %u of reference block %zu of slab %u", + i, block_index, block->slab->slab_number); + } + } + + block->allocated_count = 0; + for (index = 0; index < COUNTS_PER_BLOCK; index++) { + if (counters[index] != EMPTY_REFERENCE_COUNT) + block->allocated_count++; + } +} + +/** + * finish_reference_block_load() - After a reference block has been read, unpack it. + * @completion: The VIO that just finished reading. + */ +static void finish_reference_block_load(struct vdo_completion *completion) +{ + struct vio *vio = as_vio(completion); + struct pooled_vio *pooled = vio_as_pooled_vio(vio); + struct reference_block *block = completion->parent; + struct vdo_slab *slab = block->slab; + + unpack_reference_block((struct packed_reference_block *) vio->data, block); + return_vio_to_pool(slab->allocator->vio_pool, pooled); + slab->active_count--; + clear_provisional_references(block); + + slab->free_blocks -= block->allocated_count; + check_if_slab_drained(slab); +} + +static void load_reference_block_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct reference_block *block = vio->completion.parent; + + continue_vio_after_io(vio, finish_reference_block_load, + block->slab->allocator->thread_id); +} + +/** + * load_reference_block() - After a block waiter has gotten a VIO from the VIO pool, load the + * block. + * @waiter: The waiter of the block to load. + * @context: The VIO returned by the pool. + */ +static void load_reference_block(struct vdo_waiter *waiter, void *context) +{ + struct pooled_vio *pooled = context; + struct vio *vio = &pooled->vio; + struct reference_block *block = + container_of(waiter, struct reference_block, waiter); + size_t block_offset = (block - block->slab->reference_blocks); + + vio->completion.parent = block; + vdo_submit_metadata_vio(vio, block->slab->ref_counts_origin + block_offset, + load_reference_block_endio, handle_io_error, + REQ_OP_READ); +} + +/** + * load_reference_blocks() - Load a slab's reference blocks from the underlying storage into a + * pre-allocated reference counter. + */ +static void load_reference_blocks(struct vdo_slab *slab) +{ + block_count_t i; + + slab->free_blocks = slab->block_count; + slab->active_count = slab->reference_block_count; + for (i = 0; i < slab->reference_block_count; i++) { + struct vdo_waiter *waiter = &slab->reference_blocks[i].waiter; + + waiter->callback = load_reference_block; + acquire_vio_from_pool(slab->allocator->vio_pool, waiter); + } +} + +/** + * drain_slab() - Drain all reference count I/O. + * + * Depending upon the type of drain being performed (as recorded in the ref_count's vdo_slab), the + * reference blocks may be loaded from disk or dirty reference blocks may be written out. + */ +static void drain_slab(struct vdo_slab *slab) +{ + bool save; + bool load; + const struct admin_state_code *state = vdo_get_admin_state_code(&slab->state); + + if (state == VDO_ADMIN_STATE_SUSPENDING) + return; + + if ((state != VDO_ADMIN_STATE_REBUILDING) && + (state != VDO_ADMIN_STATE_SAVE_FOR_SCRUBBING)) + commit_tail(&slab->journal); + + if ((state == VDO_ADMIN_STATE_RECOVERING) || (slab->counters == NULL)) + return; + + save = false; + load = slab->allocator->summary_entries[slab->slab_number].load_ref_counts; + if (state == VDO_ADMIN_STATE_SCRUBBING) { + if (load) { + load_reference_blocks(slab); + return; + } + } else if (state == VDO_ADMIN_STATE_SAVE_FOR_SCRUBBING) { + if (!load) { + /* These reference counts were never written, so mark them all dirty. */ + dirty_all_reference_blocks(slab); + } + save = true; + } else if (state == VDO_ADMIN_STATE_REBUILDING) { + /* + * Write out the counters if the slab has written them before, or it has any + * non-zero reference counts, or there are any slab journal blocks. + */ + block_count_t data_blocks = slab->allocator->depot->slab_config.data_blocks; + + if (load || (slab->free_blocks != data_blocks) || + !is_slab_journal_blank(slab)) { + dirty_all_reference_blocks(slab); + save = true; + } + } else if (state == VDO_ADMIN_STATE_SAVING) { + save = (slab->status == VDO_SLAB_REBUILT); + } else { + vdo_finish_draining_with_result(&slab->state, VDO_SUCCESS); + return; + } + + if (save) + save_dirty_reference_blocks(slab); +} + +static int allocate_slab_counters(struct vdo_slab *slab) +{ + int result; + size_t index, bytes; + + result = VDO_ASSERT(slab->reference_blocks == NULL, + "vdo_slab %u doesn't allocate refcounts twice", + slab->slab_number); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate(slab->reference_block_count, struct reference_block, + __func__, &slab->reference_blocks); + if (result != VDO_SUCCESS) + return result; + + /* + * Allocate such that the runt slab has a full-length memory array, plus a little padding + * so we can word-search even at the very end. + */ + bytes = (slab->reference_block_count * COUNTS_PER_BLOCK) + (2 * BYTES_PER_WORD); + result = vdo_allocate(bytes, vdo_refcount_t, "ref counts array", + &slab->counters); + if (result != VDO_SUCCESS) { + vdo_free(vdo_forget(slab->reference_blocks)); + return result; + } + + slab->search_cursor.first_block = slab->reference_blocks; + slab->search_cursor.last_block = &slab->reference_blocks[slab->reference_block_count - 1]; + reset_search_cursor(slab); + + for (index = 0; index < slab->reference_block_count; index++) { + slab->reference_blocks[index] = (struct reference_block) { + .slab = slab, + }; + } + + return VDO_SUCCESS; +} + +static int allocate_counters_if_clean(struct vdo_slab *slab) +{ + if (vdo_is_state_clean_load(&slab->state)) + return allocate_slab_counters(slab); + + return VDO_SUCCESS; +} + +static void finish_loading_journal(struct vdo_completion *completion) +{ + struct vio *vio = as_vio(completion); + struct slab_journal *journal = completion->parent; + struct vdo_slab *slab = journal->slab; + struct packed_slab_journal_block *block = (struct packed_slab_journal_block *) vio->data; + struct slab_journal_block_header header; + + vdo_unpack_slab_journal_block_header(&block->header, &header); + + /* FIXME: should it be an error if the following conditional fails? */ + if ((header.metadata_type == VDO_METADATA_SLAB_JOURNAL) && + (header.nonce == slab->allocator->nonce)) { + journal->tail = header.sequence_number + 1; + + /* + * If the slab is clean, this implies the slab journal is empty, so advance the + * head appropriately. + */ + journal->head = (slab->allocator->summary_entries[slab->slab_number].is_dirty ? + header.head : journal->tail); + journal->tail_header = header; + initialize_journal_state(journal); + } + + return_vio_to_pool(slab->allocator->vio_pool, vio_as_pooled_vio(vio)); + vdo_finish_loading_with_result(&slab->state, allocate_counters_if_clean(slab)); +} + +static void read_slab_journal_tail_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct slab_journal *journal = vio->completion.parent; + + continue_vio_after_io(vio, finish_loading_journal, + journal->slab->allocator->thread_id); +} + +static void handle_load_error(struct vdo_completion *completion) +{ + int result = completion->result; + struct slab_journal *journal = completion->parent; + struct vio *vio = as_vio(completion); + + vio_record_metadata_io_error(vio); + return_vio_to_pool(journal->slab->allocator->vio_pool, vio_as_pooled_vio(vio)); + vdo_finish_loading_with_result(&journal->slab->state, result); +} + +/** + * read_slab_journal_tail() - Read the slab journal tail block by using a vio acquired from the vio + * pool. + * @waiter: The vio pool waiter which has just been notified. + * @context: The vio pool entry given to the waiter. + * + * This is the success callback from acquire_vio_from_pool() when loading a slab journal. + */ +static void read_slab_journal_tail(struct vdo_waiter *waiter, void *context) +{ + struct slab_journal *journal = + container_of(waiter, struct slab_journal, resource_waiter); + struct vdo_slab *slab = journal->slab; + struct pooled_vio *pooled = context; + struct vio *vio = &pooled->vio; + tail_block_offset_t last_commit_point = + slab->allocator->summary_entries[slab->slab_number].tail_block_offset; + + /* + * Slab summary keeps the commit point offset, so the tail block is the block before that. + * Calculation supports small journals in unit tests. + */ + tail_block_offset_t tail_block = ((last_commit_point == 0) ? + (tail_block_offset_t)(journal->size - 1) : + (last_commit_point - 1)); + + vio->completion.parent = journal; + vio->completion.callback_thread_id = slab->allocator->thread_id; + vdo_submit_metadata_vio(vio, slab->journal_origin + tail_block, + read_slab_journal_tail_endio, handle_load_error, + REQ_OP_READ); +} + +/** + * load_slab_journal() - Load a slab's journal by reading the journal's tail. + */ +static void load_slab_journal(struct vdo_slab *slab) +{ + struct slab_journal *journal = &slab->journal; + tail_block_offset_t last_commit_point; + + last_commit_point = slab->allocator->summary_entries[slab->slab_number].tail_block_offset; + if ((last_commit_point == 0) && + !slab->allocator->summary_entries[slab->slab_number].load_ref_counts) { + /* + * This slab claims that it has a tail block at (journal->size - 1), but a head of + * 1. This is impossible, due to the scrubbing threshold, on a real system, so + * don't bother reading the (bogus) data off disk. + */ + VDO_ASSERT_LOG_ONLY(((journal->size < 16) || + (journal->scrubbing_threshold < (journal->size - 1))), + "Scrubbing threshold protects against reads of unwritten slab journal blocks"); + vdo_finish_loading_with_result(&slab->state, + allocate_counters_if_clean(slab)); + return; + } + + journal->resource_waiter.callback = read_slab_journal_tail; + acquire_vio_from_pool(slab->allocator->vio_pool, &journal->resource_waiter); +} + +static void register_slab_for_scrubbing(struct vdo_slab *slab, bool high_priority) +{ + struct slab_scrubber *scrubber = &slab->allocator->scrubber; + + VDO_ASSERT_LOG_ONLY((slab->status != VDO_SLAB_REBUILT), + "slab to be scrubbed is unrecovered"); + + if (slab->status != VDO_SLAB_REQUIRES_SCRUBBING) + return; + + list_del_init(&slab->allocq_entry); + if (!slab->was_queued_for_scrubbing) { + WRITE_ONCE(scrubber->slab_count, scrubber->slab_count + 1); + slab->was_queued_for_scrubbing = true; + } + + if (high_priority) { + slab->status = VDO_SLAB_REQUIRES_HIGH_PRIORITY_SCRUBBING; + list_add_tail(&slab->allocq_entry, &scrubber->high_priority_slabs); + return; + } + + list_add_tail(&slab->allocq_entry, &scrubber->slabs); +} + +/* Queue a slab for allocation or scrubbing. */ +static void queue_slab(struct vdo_slab *slab) +{ + struct block_allocator *allocator = slab->allocator; + block_count_t free_blocks; + int result; + + VDO_ASSERT_LOG_ONLY(list_empty(&slab->allocq_entry), + "a requeued slab must not already be on a ring"); + + if (vdo_is_read_only(allocator->depot->vdo)) + return; + + free_blocks = slab->free_blocks; + result = VDO_ASSERT((free_blocks <= allocator->depot->slab_config.data_blocks), + "rebuilt slab %u must have a valid free block count (has %llu, expected maximum %llu)", + slab->slab_number, (unsigned long long) free_blocks, + (unsigned long long) allocator->depot->slab_config.data_blocks); + if (result != VDO_SUCCESS) { + vdo_enter_read_only_mode(allocator->depot->vdo, result); + return; + } + + if (slab->status != VDO_SLAB_REBUILT) { + register_slab_for_scrubbing(slab, false); + return; + } + + if (!vdo_is_state_resuming(&slab->state)) { + /* + * If the slab is resuming, we've already accounted for it here, so don't do it + * again. + * FIXME: under what situation would the slab be resuming here? + */ + WRITE_ONCE(allocator->allocated_blocks, + allocator->allocated_blocks - free_blocks); + if (!is_slab_journal_blank(slab)) { + WRITE_ONCE(allocator->statistics.slabs_opened, + allocator->statistics.slabs_opened + 1); + } + } + + if (allocator->depot->vdo->suspend_type == VDO_ADMIN_STATE_SAVING) + reopen_slab_journal(slab); + + prioritize_slab(slab); +} + +/** + * initiate_slab_action() - Initiate a slab action. + * + * Implements vdo_admin_initiator_fn. + */ +static void initiate_slab_action(struct admin_state *state) +{ + struct vdo_slab *slab = container_of(state, struct vdo_slab, state); + + if (vdo_is_state_draining(state)) { + const struct admin_state_code *operation = vdo_get_admin_state_code(state); + + if (operation == VDO_ADMIN_STATE_SCRUBBING) + slab->status = VDO_SLAB_REBUILDING; + + drain_slab(slab); + check_if_slab_drained(slab); + return; + } + + if (vdo_is_state_loading(state)) { + load_slab_journal(slab); + return; + } + + if (vdo_is_state_resuming(state)) { + queue_slab(slab); + vdo_finish_resuming(state); + return; + } + + vdo_finish_operation(state, VDO_INVALID_ADMIN_STATE); +} + +/** + * get_next_slab() - Get the next slab to scrub. + * @scrubber: The slab scrubber. + * + * Return: The next slab to scrub or NULL if there are none. + */ +static struct vdo_slab *get_next_slab(struct slab_scrubber *scrubber) +{ + struct vdo_slab *slab; + + slab = list_first_entry_or_null(&scrubber->high_priority_slabs, + struct vdo_slab, allocq_entry); + if (slab != NULL) + return slab; + + return list_first_entry_or_null(&scrubber->slabs, struct vdo_slab, + allocq_entry); +} + +/** + * has_slabs_to_scrub() - Check whether a scrubber has slabs to scrub. + * @scrubber: The scrubber to check. + * + * Return: true if the scrubber has slabs to scrub. + */ +static inline bool __must_check has_slabs_to_scrub(struct slab_scrubber *scrubber) +{ + return (get_next_slab(scrubber) != NULL); +} + +/** + * uninitialize_scrubber_vio() - Clean up the slab_scrubber's vio. + * @scrubber: The scrubber. + */ +static void uninitialize_scrubber_vio(struct slab_scrubber *scrubber) +{ + vdo_free(vdo_forget(scrubber->vio.data)); + free_vio_components(&scrubber->vio); +} + +/** + * finish_scrubbing() - Stop scrubbing, either because there are no more slabs to scrub or because + * there's been an error. + * @scrubber: The scrubber. + */ +static void finish_scrubbing(struct slab_scrubber *scrubber, int result) +{ + bool notify = vdo_waitq_has_waiters(&scrubber->waiters); + bool done = !has_slabs_to_scrub(scrubber); + struct block_allocator *allocator = + container_of(scrubber, struct block_allocator, scrubber); + + if (done) + uninitialize_scrubber_vio(scrubber); + + if (scrubber->high_priority_only) { + scrubber->high_priority_only = false; + vdo_fail_completion(vdo_forget(scrubber->vio.completion.parent), result); + } else if (done && (atomic_add_return(-1, &allocator->depot->zones_to_scrub) == 0)) { + /* All of our slabs were scrubbed, and we're the last allocator to finish. */ + enum vdo_state prior_state = + atomic_cmpxchg(&allocator->depot->vdo->state, VDO_RECOVERING, + VDO_DIRTY); + + /* + * To be safe, even if the CAS failed, ensure anything that follows is ordered with + * respect to whatever state change did happen. + */ + smp_mb__after_atomic(); + + /* + * We must check the VDO state here and not the depot's read_only_notifier since + * the compare-swap-above could have failed due to a read-only entry which our own + * thread does not yet know about. + */ + if (prior_state == VDO_DIRTY) + vdo_log_info("VDO commencing normal operation"); + else if (prior_state == VDO_RECOVERING) + vdo_log_info("Exiting recovery mode"); + } + + /* + * Note that the scrubber has stopped, and inform anyone who might be waiting for that to + * happen. + */ + if (!vdo_finish_draining(&scrubber->admin_state)) + WRITE_ONCE(scrubber->admin_state.current_state, + VDO_ADMIN_STATE_SUSPENDED); + + /* + * We can't notify waiters until after we've finished draining or they'll just requeue. + * Fortunately if there were waiters, we can't have been freed yet. + */ + if (notify) + vdo_waitq_notify_all_waiters(&scrubber->waiters, NULL, NULL); +} + +static void scrub_next_slab(struct slab_scrubber *scrubber); + +/** + * slab_scrubbed() - Notify the scrubber that a slab has been scrubbed. + * @completion: The slab rebuild completion. + * + * This callback is registered in apply_journal_entries(). + */ +static void slab_scrubbed(struct vdo_completion *completion) +{ + struct slab_scrubber *scrubber = + container_of(as_vio(completion), struct slab_scrubber, vio); + struct vdo_slab *slab = scrubber->slab; + + slab->status = VDO_SLAB_REBUILT; + queue_slab(slab); + reopen_slab_journal(slab); + WRITE_ONCE(scrubber->slab_count, scrubber->slab_count - 1); + scrub_next_slab(scrubber); +} + +/** + * abort_scrubbing() - Abort scrubbing due to an error. + * @scrubber: The slab scrubber. + * @result: The error. + */ +static void abort_scrubbing(struct slab_scrubber *scrubber, int result) +{ + vdo_enter_read_only_mode(scrubber->vio.completion.vdo, result); + finish_scrubbing(scrubber, result); +} + +/** + * handle_scrubber_error() - Handle errors while rebuilding a slab. + * @completion: The slab rebuild completion. + */ +static void handle_scrubber_error(struct vdo_completion *completion) +{ + struct vio *vio = as_vio(completion); + + vio_record_metadata_io_error(vio); + abort_scrubbing(container_of(vio, struct slab_scrubber, vio), + completion->result); +} + +/** + * apply_block_entries() - Apply all the entries in a block to the reference counts. + * @block: A block with entries to apply. + * @entry_count: The number of entries to apply. + * @block_number: The sequence number of the block. + * @slab: The slab to apply the entries to. + * + * Return: VDO_SUCCESS or an error code. + */ +static int apply_block_entries(struct packed_slab_journal_block *block, + journal_entry_count_t entry_count, + sequence_number_t block_number, struct vdo_slab *slab) +{ + struct journal_point entry_point = { + .sequence_number = block_number, + .entry_count = 0, + }; + int result; + slab_block_number max_sbn = slab->end - slab->start; + + while (entry_point.entry_count < entry_count) { + struct slab_journal_entry entry = + vdo_decode_slab_journal_entry(block, entry_point.entry_count); + + if (entry.sbn > max_sbn) { + /* This entry is out of bounds. */ + return vdo_log_error_strerror(VDO_CORRUPT_JOURNAL, + "vdo_slab journal entry (%llu, %u) had invalid offset %u in slab (size %u blocks)", + (unsigned long long) block_number, + entry_point.entry_count, + entry.sbn, max_sbn); + } + + result = replay_reference_count_change(slab, &entry_point, entry); + if (result != VDO_SUCCESS) { + vdo_log_error_strerror(result, + "vdo_slab journal entry (%llu, %u) (%s of offset %u) could not be applied in slab %u", + (unsigned long long) block_number, + entry_point.entry_count, + vdo_get_journal_operation_name(entry.operation), + entry.sbn, slab->slab_number); + return result; + } + entry_point.entry_count++; + } + + return VDO_SUCCESS; +} + +/** + * apply_journal_entries() - Find the relevant vio of the slab journal and apply all valid entries. + * @completion: The metadata read vio completion. + * + * This is a callback registered in start_scrubbing(). + */ +static void apply_journal_entries(struct vdo_completion *completion) +{ + int result; + struct slab_scrubber *scrubber = + container_of(as_vio(completion), struct slab_scrubber, vio); + struct vdo_slab *slab = scrubber->slab; + struct slab_journal *journal = &slab->journal; + + /* Find the boundaries of the useful part of the journal. */ + sequence_number_t tail = journal->tail; + tail_block_offset_t end_index = (tail - 1) % journal->size; + char *end_data = scrubber->vio.data + (end_index * VDO_BLOCK_SIZE); + struct packed_slab_journal_block *end_block = + (struct packed_slab_journal_block *) end_data; + + sequence_number_t head = __le64_to_cpu(end_block->header.head); + tail_block_offset_t head_index = head % journal->size; + block_count_t index = head_index; + + struct journal_point ref_counts_point = slab->slab_journal_point; + struct journal_point last_entry_applied = ref_counts_point; + sequence_number_t sequence; + + for (sequence = head; sequence < tail; sequence++) { + char *block_data = scrubber->vio.data + (index * VDO_BLOCK_SIZE); + struct packed_slab_journal_block *block = + (struct packed_slab_journal_block *) block_data; + struct slab_journal_block_header header; + + vdo_unpack_slab_journal_block_header(&block->header, &header); + + if ((header.nonce != slab->allocator->nonce) || + (header.metadata_type != VDO_METADATA_SLAB_JOURNAL) || + (header.sequence_number != sequence) || + (header.entry_count > journal->entries_per_block) || + (header.has_block_map_increments && + (header.entry_count > journal->full_entries_per_block))) { + /* The block is not what we expect it to be. */ + vdo_log_error("vdo_slab journal block for slab %u was invalid", + slab->slab_number); + abort_scrubbing(scrubber, VDO_CORRUPT_JOURNAL); + return; + } + + result = apply_block_entries(block, header.entry_count, sequence, slab); + if (result != VDO_SUCCESS) { + abort_scrubbing(scrubber, result); + return; + } + + last_entry_applied.sequence_number = sequence; + last_entry_applied.entry_count = header.entry_count - 1; + index++; + if (index == journal->size) + index = 0; + } + + /* + * At the end of rebuild, the reference counters should be accurate to the end of the + * journal we just applied. + */ + result = VDO_ASSERT(!vdo_before_journal_point(&last_entry_applied, + &ref_counts_point), + "Refcounts are not more accurate than the slab journal"); + if (result != VDO_SUCCESS) { + abort_scrubbing(scrubber, result); + return; + } + + /* Save out the rebuilt reference blocks. */ + vdo_prepare_completion(completion, slab_scrubbed, handle_scrubber_error, + slab->allocator->thread_id, completion->parent); + vdo_start_operation_with_waiter(&slab->state, + VDO_ADMIN_STATE_SAVE_FOR_SCRUBBING, + completion, initiate_slab_action); +} + +static void read_slab_journal_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct slab_scrubber *scrubber = container_of(vio, struct slab_scrubber, vio); + + continue_vio_after_io(bio->bi_private, apply_journal_entries, + scrubber->slab->allocator->thread_id); +} + +/** + * start_scrubbing() - Read the current slab's journal from disk now that it has been flushed. + * @completion: The scrubber's vio completion. + * + * This callback is registered in scrub_next_slab(). + */ +static void start_scrubbing(struct vdo_completion *completion) +{ + struct slab_scrubber *scrubber = + container_of(as_vio(completion), struct slab_scrubber, vio); + struct vdo_slab *slab = scrubber->slab; + + if (!slab->allocator->summary_entries[slab->slab_number].is_dirty) { + slab_scrubbed(completion); + return; + } + + vdo_submit_metadata_vio(&scrubber->vio, slab->journal_origin, + read_slab_journal_endio, handle_scrubber_error, + REQ_OP_READ); +} + +/** + * scrub_next_slab() - Scrub the next slab if there is one. + * @scrubber: The scrubber. + */ +static void scrub_next_slab(struct slab_scrubber *scrubber) +{ + struct vdo_completion *completion = &scrubber->vio.completion; + struct vdo_slab *slab; + + /* + * Note: this notify call is always safe only because scrubbing can only be started when + * the VDO is quiescent. + */ + vdo_waitq_notify_all_waiters(&scrubber->waiters, NULL, NULL); + + if (vdo_is_read_only(completion->vdo)) { + finish_scrubbing(scrubber, VDO_READ_ONLY); + return; + } + + slab = get_next_slab(scrubber); + if ((slab == NULL) || + (scrubber->high_priority_only && list_empty(&scrubber->high_priority_slabs))) { + finish_scrubbing(scrubber, VDO_SUCCESS); + return; + } + + if (vdo_finish_draining(&scrubber->admin_state)) + return; + + list_del_init(&slab->allocq_entry); + scrubber->slab = slab; + vdo_prepare_completion(completion, start_scrubbing, handle_scrubber_error, + slab->allocator->thread_id, completion->parent); + vdo_start_operation_with_waiter(&slab->state, VDO_ADMIN_STATE_SCRUBBING, + completion, initiate_slab_action); +} + +/** + * scrub_slabs() - Scrub all of an allocator's slabs that are eligible for scrubbing. + * @allocator: The block_allocator to scrub. + * @parent: The completion to notify when scrubbing is done, implies high_priority, may be NULL. + */ +static void scrub_slabs(struct block_allocator *allocator, struct vdo_completion *parent) +{ + struct slab_scrubber *scrubber = &allocator->scrubber; + + scrubber->vio.completion.parent = parent; + scrubber->high_priority_only = (parent != NULL); + if (!has_slabs_to_scrub(scrubber)) { + finish_scrubbing(scrubber, VDO_SUCCESS); + return; + } + + if (scrubber->high_priority_only && + vdo_is_priority_table_empty(allocator->prioritized_slabs) && + list_empty(&scrubber->high_priority_slabs)) + register_slab_for_scrubbing(get_next_slab(scrubber), true); + + vdo_resume_if_quiescent(&scrubber->admin_state); + scrub_next_slab(scrubber); +} + +static inline void assert_on_allocator_thread(thread_id_t thread_id, + const char *function_name) +{ + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == thread_id), + "%s called on correct thread", function_name); +} + +static void register_slab_with_allocator(struct block_allocator *allocator, + struct vdo_slab *slab) +{ + allocator->slab_count++; + allocator->last_slab = slab->slab_number; +} + +/** + * get_depot_slab_iterator() - Return a slab_iterator over the slabs in a slab_depot. + * @depot: The depot over which to iterate. + * @start: The number of the slab to start iterating from. + * @end: The number of the last slab which may be returned. + * @stride: The difference in slab number between successive slabs. + * + * Iteration always occurs from higher to lower numbered slabs. + * + * Return: An initialized iterator structure. + */ +static struct slab_iterator get_depot_slab_iterator(struct slab_depot *depot, + slab_count_t start, slab_count_t end, + slab_count_t stride) +{ + struct vdo_slab **slabs = depot->slabs; + + return (struct slab_iterator) { + .slabs = slabs, + .next = (((slabs == NULL) || (start < end)) ? NULL : slabs[start]), + .end = end, + .stride = stride, + }; +} + +static struct slab_iterator get_slab_iterator(const struct block_allocator *allocator) +{ + return get_depot_slab_iterator(allocator->depot, allocator->last_slab, + allocator->zone_number, + allocator->depot->zone_count); +} + +/** + * next_slab() - Get the next slab from a slab_iterator and advance the iterator + * @iterator: The slab_iterator. + * + * Return: The next slab or NULL if the iterator is exhausted. + */ +static struct vdo_slab *next_slab(struct slab_iterator *iterator) +{ + struct vdo_slab *slab = iterator->next; + + if ((slab == NULL) || (slab->slab_number < iterator->end + iterator->stride)) + iterator->next = NULL; + else + iterator->next = iterator->slabs[slab->slab_number - iterator->stride]; + + return slab; +} + +/** + * abort_waiter() - Abort vios waiting to make journal entries when read-only. + * + * This callback is invoked on all vios waiting to make slab journal entries after the VDO has gone + * into read-only mode. Implements waiter_callback_fn. + */ +static void abort_waiter(struct vdo_waiter *waiter, void *context __always_unused) +{ + struct reference_updater *updater = + container_of(waiter, struct reference_updater, waiter); + struct data_vio *data_vio = data_vio_from_reference_updater(updater); + + if (updater->increment) { + continue_data_vio_with_error(data_vio, VDO_READ_ONLY); + return; + } + + vdo_continue_completion(&data_vio->decrement_completion, VDO_READ_ONLY); +} + +/* Implements vdo_read_only_notification_fn. */ +static void notify_block_allocator_of_read_only_mode(void *listener, + struct vdo_completion *parent) +{ + struct block_allocator *allocator = listener; + struct slab_iterator iterator; + + assert_on_allocator_thread(allocator->thread_id, __func__); + iterator = get_slab_iterator(allocator); + while (iterator.next != NULL) { + struct vdo_slab *slab = next_slab(&iterator); + + vdo_waitq_notify_all_waiters(&slab->journal.entry_waiters, + abort_waiter, &slab->journal); + check_if_slab_drained(slab); + } + + vdo_finish_completion(parent); +} + +/** + * vdo_acquire_provisional_reference() - Acquire a provisional reference on behalf of a PBN lock if + * the block it locks is unreferenced. + * @slab: The slab which contains the block. + * @pbn: The physical block to reference. + * @lock: The lock. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_acquire_provisional_reference(struct vdo_slab *slab, physical_block_number_t pbn, + struct pbn_lock *lock) +{ + slab_block_number block_number; + int result; + + if (vdo_pbn_lock_has_provisional_reference(lock)) + return VDO_SUCCESS; + + if (!is_slab_open(slab)) + return VDO_INVALID_ADMIN_STATE; + + result = slab_block_number_from_pbn(slab, pbn, &block_number); + if (result != VDO_SUCCESS) + return result; + + if (slab->counters[block_number] == EMPTY_REFERENCE_COUNT) { + make_provisional_reference(slab, block_number); + if (lock != NULL) + vdo_assign_pbn_lock_provisional_reference(lock); + } + + if (vdo_pbn_lock_has_provisional_reference(lock)) + adjust_free_block_count(slab, false); + + return VDO_SUCCESS; +} + +static int __must_check allocate_slab_block(struct vdo_slab *slab, + physical_block_number_t *block_number_ptr) +{ + slab_block_number free_index; + + if (!is_slab_open(slab)) + return VDO_INVALID_ADMIN_STATE; + + if (!search_reference_blocks(slab, &free_index)) + return VDO_NO_SPACE; + + VDO_ASSERT_LOG_ONLY((slab->counters[free_index] == EMPTY_REFERENCE_COUNT), + "free block must have ref count of zero"); + make_provisional_reference(slab, free_index); + adjust_free_block_count(slab, false); + + /* + * Update the search hint so the next search will start at the array index just past the + * free block we just found. + */ + slab->search_cursor.index = (free_index + 1); + + *block_number_ptr = slab->start + free_index; + return VDO_SUCCESS; +} + +/** + * open_slab() - Prepare a slab to be allocated from. + * @slab: The slab. + */ +static void open_slab(struct vdo_slab *slab) +{ + reset_search_cursor(slab); + if (is_slab_journal_blank(slab)) { + WRITE_ONCE(slab->allocator->statistics.slabs_opened, + slab->allocator->statistics.slabs_opened + 1); + dirty_all_reference_blocks(slab); + } else { + WRITE_ONCE(slab->allocator->statistics.slabs_reopened, + slab->allocator->statistics.slabs_reopened + 1); + } + + slab->allocator->open_slab = slab; +} + + +/* + * The block allocated will have a provisional reference and the reference must be either confirmed + * with a subsequent increment or vacated with a subsequent decrement via + * vdo_release_block_reference(). + */ +int vdo_allocate_block(struct block_allocator *allocator, + physical_block_number_t *block_number_ptr) +{ + int result; + + if (allocator->open_slab != NULL) { + /* Try to allocate the next block in the currently open slab. */ + result = allocate_slab_block(allocator->open_slab, block_number_ptr); + if ((result == VDO_SUCCESS) || (result != VDO_NO_SPACE)) + return result; + + /* Put the exhausted open slab back into the priority table. */ + prioritize_slab(allocator->open_slab); + } + + /* Remove the highest priority slab from the priority table and make it the open slab. */ + open_slab(list_entry(vdo_priority_table_dequeue(allocator->prioritized_slabs), + struct vdo_slab, allocq_entry)); + + /* + * Try allocating again. If we're out of space immediately after opening a slab, then every + * slab must be fully allocated. + */ + return allocate_slab_block(allocator->open_slab, block_number_ptr); +} + +/** + * vdo_enqueue_clean_slab_waiter() - Wait for a clean slab. + * @allocator: The block_allocator on which to wait. + * @waiter: The waiter. + * + * Return: VDO_SUCCESS if the waiter was queued, VDO_NO_SPACE if there are no slabs to scrub, and + * some other error otherwise. + */ +int vdo_enqueue_clean_slab_waiter(struct block_allocator *allocator, + struct vdo_waiter *waiter) +{ + if (vdo_is_read_only(allocator->depot->vdo)) + return VDO_READ_ONLY; + + if (vdo_is_state_quiescent(&allocator->scrubber.admin_state)) + return VDO_NO_SPACE; + + vdo_waitq_enqueue_waiter(&allocator->scrubber.waiters, waiter); + return VDO_SUCCESS; +} + +/** + * vdo_modify_reference_count() - Modify the reference count of a block by first making a slab + * journal entry and then updating the reference counter. + * + * @data_vio: The data_vio for which to add the entry. + * @updater: Which of the data_vio's reference updaters is being submitted. + */ +void vdo_modify_reference_count(struct vdo_completion *completion, + struct reference_updater *updater) +{ + struct vdo_slab *slab = vdo_get_slab(completion->vdo->depot, updater->zpbn.pbn); + + if (!is_slab_open(slab)) { + vdo_continue_completion(completion, VDO_INVALID_ADMIN_STATE); + return; + } + + if (vdo_is_read_only(completion->vdo)) { + vdo_continue_completion(completion, VDO_READ_ONLY); + return; + } + + vdo_waitq_enqueue_waiter(&slab->journal.entry_waiters, &updater->waiter); + if ((slab->status != VDO_SLAB_REBUILT) && requires_reaping(&slab->journal)) + register_slab_for_scrubbing(slab, true); + + add_entries(&slab->journal); +} + +/* Release an unused provisional reference. */ +int vdo_release_block_reference(struct block_allocator *allocator, + physical_block_number_t pbn) +{ + struct reference_updater updater; + + if (pbn == VDO_ZERO_BLOCK) + return VDO_SUCCESS; + + updater = (struct reference_updater) { + .operation = VDO_JOURNAL_DATA_REMAPPING, + .increment = false, + .zpbn = { + .pbn = pbn, + }, + }; + + return adjust_reference_count(vdo_get_slab(allocator->depot, pbn), + &updater, NULL); +} + +/* + * This is a min_heap callback function orders slab_status structures using the 'is_clean' field as + * the primary key and the 'emptiness' field as the secondary key. + * + * Slabs need to be pushed onto the rings in the same order they are to be popped off. Popping + * should always get the most empty first, so pushing should be from most empty to least empty. + * Thus, the ordering is reversed from the usual sense since min_heap returns smaller elements + * before larger ones. + */ +static bool slab_status_is_less_than(const void *item1, const void *item2) +{ + const struct slab_status *info1 = item1; + const struct slab_status *info2 = item2; + + if (info1->is_clean != info2->is_clean) + return info1->is_clean; + if (info1->emptiness != info2->emptiness) + return info1->emptiness > info2->emptiness; + return info1->slab_number < info2->slab_number; +} + +static void swap_slab_statuses(void *item1, void *item2) +{ + struct slab_status *info1 = item1; + struct slab_status *info2 = item2; + + swap(*info1, *info2); +} + +static const struct min_heap_callbacks slab_status_min_heap = { + .elem_size = sizeof(struct slab_status), + .less = slab_status_is_less_than, + .swp = swap_slab_statuses, +}; + +/* Inform the slab actor that a action has finished on some slab; used by apply_to_slabs(). */ +static void slab_action_callback(struct vdo_completion *completion) +{ + struct block_allocator *allocator = vdo_as_block_allocator(completion); + struct slab_actor *actor = &allocator->slab_actor; + + if (--actor->slab_action_count == 0) { + actor->callback(completion); + return; + } + + vdo_reset_completion(completion); +} + +/* Preserve the error from part of an action and continue. */ +static void handle_operation_error(struct vdo_completion *completion) +{ + struct block_allocator *allocator = vdo_as_block_allocator(completion); + + if (allocator->state.waiter != NULL) + vdo_set_completion_result(allocator->state.waiter, completion->result); + completion->callback(completion); +} + +/* Perform an action on each of an allocator's slabs in parallel. */ +static void apply_to_slabs(struct block_allocator *allocator, vdo_action_fn callback) +{ + struct slab_iterator iterator; + + vdo_prepare_completion(&allocator->completion, slab_action_callback, + handle_operation_error, allocator->thread_id, NULL); + allocator->completion.requeue = false; + + /* + * Since we are going to dequeue all of the slabs, the open slab will become invalid, so + * clear it. + */ + allocator->open_slab = NULL; + + /* Ensure that we don't finish before we're done starting. */ + allocator->slab_actor = (struct slab_actor) { + .slab_action_count = 1, + .callback = callback, + }; + + iterator = get_slab_iterator(allocator); + while (iterator.next != NULL) { + const struct admin_state_code *operation = + vdo_get_admin_state_code(&allocator->state); + struct vdo_slab *slab = next_slab(&iterator); + + list_del_init(&slab->allocq_entry); + allocator->slab_actor.slab_action_count++; + vdo_start_operation_with_waiter(&slab->state, operation, + &allocator->completion, + initiate_slab_action); + } + + slab_action_callback(&allocator->completion); +} + +static void finish_loading_allocator(struct vdo_completion *completion) +{ + struct block_allocator *allocator = vdo_as_block_allocator(completion); + const struct admin_state_code *operation = + vdo_get_admin_state_code(&allocator->state); + + if (allocator->eraser != NULL) + dm_kcopyd_client_destroy(vdo_forget(allocator->eraser)); + + if (operation == VDO_ADMIN_STATE_LOADING_FOR_RECOVERY) { + void *context = + vdo_get_current_action_context(allocator->depot->action_manager); + + vdo_replay_into_slab_journals(allocator, context); + return; + } + + vdo_finish_loading(&allocator->state); +} + +static void erase_next_slab_journal(struct block_allocator *allocator); + +static void copy_callback(int read_err, unsigned long write_err, void *context) +{ + struct block_allocator *allocator = context; + int result = (((read_err == 0) && (write_err == 0)) ? VDO_SUCCESS : -EIO); + + if (result != VDO_SUCCESS) { + vdo_fail_completion(&allocator->completion, result); + return; + } + + erase_next_slab_journal(allocator); +} + +/* erase_next_slab_journal() - Erase the next slab journal. */ +static void erase_next_slab_journal(struct block_allocator *allocator) +{ + struct vdo_slab *slab; + physical_block_number_t pbn; + struct dm_io_region regions[1]; + struct slab_depot *depot = allocator->depot; + block_count_t blocks = depot->slab_config.slab_journal_blocks; + + if (allocator->slabs_to_erase.next == NULL) { + vdo_finish_completion(&allocator->completion); + return; + } + + slab = next_slab(&allocator->slabs_to_erase); + pbn = slab->journal_origin - depot->vdo->geometry.bio_offset; + regions[0] = (struct dm_io_region) { + .bdev = vdo_get_backing_device(depot->vdo), + .sector = pbn * VDO_SECTORS_PER_BLOCK, + .count = blocks * VDO_SECTORS_PER_BLOCK, + }; + dm_kcopyd_zero(allocator->eraser, 1, regions, 0, copy_callback, allocator); +} + +/* Implements vdo_admin_initiator_fn. */ +static void initiate_load(struct admin_state *state) +{ + struct block_allocator *allocator = + container_of(state, struct block_allocator, state); + const struct admin_state_code *operation = vdo_get_admin_state_code(state); + + if (operation == VDO_ADMIN_STATE_LOADING_FOR_REBUILD) { + /* + * Must requeue because the kcopyd client cannot be freed in the same stack frame + * as the kcopyd callback, lest it deadlock. + */ + vdo_prepare_completion_for_requeue(&allocator->completion, + finish_loading_allocator, + handle_operation_error, + allocator->thread_id, NULL); + allocator->eraser = dm_kcopyd_client_create(NULL); + if (IS_ERR(allocator->eraser)) { + vdo_fail_completion(&allocator->completion, + PTR_ERR(allocator->eraser)); + allocator->eraser = NULL; + return; + } + allocator->slabs_to_erase = get_slab_iterator(allocator); + + erase_next_slab_journal(allocator); + return; + } + + apply_to_slabs(allocator, finish_loading_allocator); +} + +/** + * vdo_notify_slab_journals_are_recovered() - Inform a block allocator that its slab journals have + * been recovered from the recovery journal. + * @completion The allocator completion + */ +void vdo_notify_slab_journals_are_recovered(struct vdo_completion *completion) +{ + struct block_allocator *allocator = vdo_as_block_allocator(completion); + + vdo_finish_loading_with_result(&allocator->state, completion->result); +} + +static int get_slab_statuses(struct block_allocator *allocator, + struct slab_status **statuses_ptr) +{ + int result; + struct slab_status *statuses; + struct slab_iterator iterator = get_slab_iterator(allocator); + + result = vdo_allocate(allocator->slab_count, struct slab_status, __func__, + &statuses); + if (result != VDO_SUCCESS) + return result; + + *statuses_ptr = statuses; + + while (iterator.next != NULL) { + slab_count_t slab_number = next_slab(&iterator)->slab_number; + + *statuses++ = (struct slab_status) { + .slab_number = slab_number, + .is_clean = !allocator->summary_entries[slab_number].is_dirty, + .emptiness = allocator->summary_entries[slab_number].fullness_hint, + }; + } + + return VDO_SUCCESS; +} + +/* Prepare slabs for allocation or scrubbing. */ +static int __must_check vdo_prepare_slabs_for_allocation(struct block_allocator *allocator) +{ + struct slab_status current_slab_status; + struct min_heap heap; + int result; + struct slab_status *slab_statuses; + struct slab_depot *depot = allocator->depot; + + WRITE_ONCE(allocator->allocated_blocks, + allocator->slab_count * depot->slab_config.data_blocks); + result = get_slab_statuses(allocator, &slab_statuses); + if (result != VDO_SUCCESS) + return result; + + /* Sort the slabs by cleanliness, then by emptiness hint. */ + heap = (struct min_heap) { + .data = slab_statuses, + .nr = allocator->slab_count, + .size = allocator->slab_count, + }; + min_heapify_all(&heap, &slab_status_min_heap); + + while (heap.nr > 0) { + bool high_priority; + struct vdo_slab *slab; + struct slab_journal *journal; + + current_slab_status = slab_statuses[0]; + min_heap_pop(&heap, &slab_status_min_heap); + slab = depot->slabs[current_slab_status.slab_number]; + + if ((depot->load_type == VDO_SLAB_DEPOT_REBUILD_LOAD) || + (!allocator->summary_entries[slab->slab_number].load_ref_counts && + current_slab_status.is_clean)) { + queue_slab(slab); + continue; + } + + slab->status = VDO_SLAB_REQUIRES_SCRUBBING; + journal = &slab->journal; + high_priority = ((current_slab_status.is_clean && + (depot->load_type == VDO_SLAB_DEPOT_NORMAL_LOAD)) || + (journal_length(journal) >= journal->scrubbing_threshold)); + register_slab_for_scrubbing(slab, high_priority); + } + + vdo_free(slab_statuses); + return VDO_SUCCESS; +} + +static const char *status_to_string(enum slab_rebuild_status status) +{ + switch (status) { + case VDO_SLAB_REBUILT: + return "REBUILT"; + case VDO_SLAB_REQUIRES_SCRUBBING: + return "SCRUBBING"; + case VDO_SLAB_REQUIRES_HIGH_PRIORITY_SCRUBBING: + return "PRIORITY_SCRUBBING"; + case VDO_SLAB_REBUILDING: + return "REBUILDING"; + case VDO_SLAB_REPLAYING: + return "REPLAYING"; + default: + return "UNKNOWN"; + } +} + +void vdo_dump_block_allocator(const struct block_allocator *allocator) +{ + unsigned int pause_counter = 0; + struct slab_iterator iterator = get_slab_iterator(allocator); + const struct slab_scrubber *scrubber = &allocator->scrubber; + + vdo_log_info("block_allocator zone %u", allocator->zone_number); + while (iterator.next != NULL) { + struct vdo_slab *slab = next_slab(&iterator); + struct slab_journal *journal = &slab->journal; + + if (slab->reference_blocks != NULL) { + /* Terse because there are a lot of slabs to dump and syslog is lossy. */ + vdo_log_info("slab %u: P%u, %llu free", slab->slab_number, + slab->priority, + (unsigned long long) slab->free_blocks); + } else { + vdo_log_info("slab %u: status %s", slab->slab_number, + status_to_string(slab->status)); + } + + vdo_log_info(" slab journal: entry_waiters=%zu waiting_to_commit=%s updating_slab_summary=%s head=%llu unreapable=%llu tail=%llu next_commit=%llu summarized=%llu last_summarized=%llu recovery_lock=%llu dirty=%s", + vdo_waitq_num_waiters(&journal->entry_waiters), + vdo_bool_to_string(journal->waiting_to_commit), + vdo_bool_to_string(journal->updating_slab_summary), + (unsigned long long) journal->head, + (unsigned long long) journal->unreapable, + (unsigned long long) journal->tail, + (unsigned long long) journal->next_commit, + (unsigned long long) journal->summarized, + (unsigned long long) journal->last_summarized, + (unsigned long long) journal->recovery_lock, + vdo_bool_to_string(journal->recovery_lock != 0)); + /* + * Given the frequency with which the locks are just a tiny bit off, it might be + * worth dumping all the locks, but that might be too much logging. + */ + + if (slab->counters != NULL) { + /* Terse because there are a lot of slabs to dump and syslog is lossy. */ + vdo_log_info(" slab: free=%u/%u blocks=%u dirty=%zu active=%zu journal@(%llu,%u)", + slab->free_blocks, slab->block_count, + slab->reference_block_count, + vdo_waitq_num_waiters(&slab->dirty_blocks), + slab->active_count, + (unsigned long long) slab->slab_journal_point.sequence_number, + slab->slab_journal_point.entry_count); + } else { + vdo_log_info(" no counters"); + } + + /* + * Wait for a while after each batch of 32 slabs dumped, an arbitrary number, + * allowing the kernel log a chance to be flushed instead of being overrun. + */ + if (pause_counter++ == 31) { + pause_counter = 0; + vdo_pause_for_logger(); + } + } + + vdo_log_info("slab_scrubber slab_count %u waiters %zu %s%s", + READ_ONCE(scrubber->slab_count), + vdo_waitq_num_waiters(&scrubber->waiters), + vdo_get_admin_state_code(&scrubber->admin_state)->name, + scrubber->high_priority_only ? ", high_priority_only " : ""); +} + +static void free_slab(struct vdo_slab *slab) +{ + if (slab == NULL) + return; + + list_del(&slab->allocq_entry); + vdo_free(vdo_forget(slab->journal.block)); + vdo_free(vdo_forget(slab->journal.locks)); + vdo_free(vdo_forget(slab->counters)); + vdo_free(vdo_forget(slab->reference_blocks)); + vdo_free(slab); +} + +static int initialize_slab_journal(struct vdo_slab *slab) +{ + struct slab_journal *journal = &slab->journal; + const struct slab_config *slab_config = &slab->allocator->depot->slab_config; + int result; + + result = vdo_allocate(slab_config->slab_journal_blocks, struct journal_lock, + __func__, &journal->locks); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate(VDO_BLOCK_SIZE, char, "struct packed_slab_journal_block", + (char **) &journal->block); + if (result != VDO_SUCCESS) + return result; + + journal->slab = slab; + journal->size = slab_config->slab_journal_blocks; + journal->flushing_threshold = slab_config->slab_journal_flushing_threshold; + journal->blocking_threshold = slab_config->slab_journal_blocking_threshold; + journal->scrubbing_threshold = slab_config->slab_journal_scrubbing_threshold; + journal->entries_per_block = VDO_SLAB_JOURNAL_ENTRIES_PER_BLOCK; + journal->full_entries_per_block = VDO_SLAB_JOURNAL_FULL_ENTRIES_PER_BLOCK; + journal->events = &slab->allocator->slab_journal_statistics; + journal->recovery_journal = slab->allocator->depot->vdo->recovery_journal; + journal->tail = 1; + journal->head = 1; + + journal->flushing_deadline = journal->flushing_threshold; + /* + * Set there to be some time between the deadline and the blocking threshold, so that + * hopefully all are done before blocking. + */ + if ((journal->blocking_threshold - journal->flushing_threshold) > 5) + journal->flushing_deadline = journal->blocking_threshold - 5; + + journal->slab_summary_waiter.callback = release_journal_locks; + + INIT_LIST_HEAD(&journal->dirty_entry); + INIT_LIST_HEAD(&journal->uncommitted_blocks); + + journal->tail_header.nonce = slab->allocator->nonce; + journal->tail_header.metadata_type = VDO_METADATA_SLAB_JOURNAL; + initialize_journal_state(journal); + return VDO_SUCCESS; +} + +/** + * make_slab() - Construct a new, empty slab. + * @slab_origin: The physical block number within the block allocator partition of the first block + * in the slab. + * @allocator: The block allocator to which the slab belongs. + * @slab_number: The slab number of the slab. + * @is_new: true if this slab is being allocated as part of a resize. + * @slab_ptr: A pointer to receive the new slab. + * + * Return: VDO_SUCCESS or an error code. + */ +static int __must_check make_slab(physical_block_number_t slab_origin, + struct block_allocator *allocator, + slab_count_t slab_number, bool is_new, + struct vdo_slab **slab_ptr) +{ + const struct slab_config *slab_config = &allocator->depot->slab_config; + struct vdo_slab *slab; + int result; + + result = vdo_allocate(1, struct vdo_slab, __func__, &slab); + if (result != VDO_SUCCESS) + return result; + + *slab = (struct vdo_slab) { + .allocator = allocator, + .start = slab_origin, + .end = slab_origin + slab_config->slab_blocks, + .slab_number = slab_number, + .ref_counts_origin = slab_origin + slab_config->data_blocks, + .journal_origin = + vdo_get_slab_journal_start_block(slab_config, slab_origin), + .block_count = slab_config->data_blocks, + .free_blocks = slab_config->data_blocks, + .reference_block_count = + vdo_get_saved_reference_count_size(slab_config->data_blocks), + }; + INIT_LIST_HEAD(&slab->allocq_entry); + + result = initialize_slab_journal(slab); + if (result != VDO_SUCCESS) { + free_slab(slab); + return result; + } + + if (is_new) { + vdo_set_admin_state_code(&slab->state, VDO_ADMIN_STATE_NEW); + result = allocate_slab_counters(slab); + if (result != VDO_SUCCESS) { + free_slab(slab); + return result; + } + } else { + vdo_set_admin_state_code(&slab->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + } + + *slab_ptr = slab; + return VDO_SUCCESS; +} + +/** + * allocate_slabs() - Allocate a new slab pointer array. + * @depot: The depot. + * @slab_count: The number of slabs the depot should have in the new array. + * + * Any existing slab pointers will be copied into the new array, and slabs will be allocated as + * needed. The newly allocated slabs will not be distributed for use by the block allocators. + * + * Return: VDO_SUCCESS or an error code. + */ +static int allocate_slabs(struct slab_depot *depot, slab_count_t slab_count) +{ + block_count_t slab_size; + bool resizing = false; + physical_block_number_t slab_origin; + int result; + + result = vdo_allocate(slab_count, struct vdo_slab *, + "slab pointer array", &depot->new_slabs); + if (result != VDO_SUCCESS) + return result; + + if (depot->slabs != NULL) { + memcpy(depot->new_slabs, depot->slabs, + depot->slab_count * sizeof(struct vdo_slab *)); + resizing = true; + } + + slab_size = depot->slab_config.slab_blocks; + slab_origin = depot->first_block + (depot->slab_count * slab_size); + + for (depot->new_slab_count = depot->slab_count; + depot->new_slab_count < slab_count; + depot->new_slab_count++, slab_origin += slab_size) { + struct block_allocator *allocator = + &depot->allocators[depot->new_slab_count % depot->zone_count]; + struct vdo_slab **slab_ptr = &depot->new_slabs[depot->new_slab_count]; + + result = make_slab(slab_origin, allocator, depot->new_slab_count, + resizing, slab_ptr); + if (result != VDO_SUCCESS) + return result; + } + + return VDO_SUCCESS; +} + +/** + * vdo_abandon_new_slabs() - Abandon any new slabs in this depot, freeing them as needed. + * @depot: The depot. + */ +void vdo_abandon_new_slabs(struct slab_depot *depot) +{ + slab_count_t i; + + if (depot->new_slabs == NULL) + return; + + for (i = depot->slab_count; i < depot->new_slab_count; i++) + free_slab(vdo_forget(depot->new_slabs[i])); + depot->new_slab_count = 0; + depot->new_size = 0; + vdo_free(vdo_forget(depot->new_slabs)); +} + +/** + * get_allocator_thread_id() - Get the ID of the thread on which a given allocator operates. + * + * Implements vdo_zone_thread_getter_fn. + */ +static thread_id_t get_allocator_thread_id(void *context, zone_count_t zone_number) +{ + return ((struct slab_depot *) context)->allocators[zone_number].thread_id; +} + +/** + * release_recovery_journal_lock() - Request the slab journal to release the recovery journal lock + * it may hold on a specified recovery journal block. + * @journal: The slab journal. + * @recovery_lock: The sequence number of the recovery journal block whose locks should be + * released. + * + * Return: true if the journal does hold a lock on the specified block (which it will release). + */ +static bool __must_check release_recovery_journal_lock(struct slab_journal *journal, + sequence_number_t recovery_lock) +{ + if (recovery_lock > journal->recovery_lock) { + VDO_ASSERT_LOG_ONLY((recovery_lock < journal->recovery_lock), + "slab journal recovery lock is not older than the recovery journal head"); + return false; + } + + if ((recovery_lock < journal->recovery_lock) || + vdo_is_read_only(journal->slab->allocator->depot->vdo)) + return false; + + /* All locks are held by the block which is in progress; write it. */ + commit_tail(journal); + return true; +} + +/* + * Request a commit of all dirty tail blocks which are locking the recovery journal block the depot + * is seeking to release. + * + * Implements vdo_zone_action_fn. + */ +static void release_tail_block_locks(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct slab_journal *journal, *tmp; + struct slab_depot *depot = context; + struct list_head *list = &depot->allocators[zone_number].dirty_slab_journals; + + list_for_each_entry_safe(journal, tmp, list, dirty_entry) { + if (!release_recovery_journal_lock(journal, + depot->active_release_request)) + break; + } + + vdo_finish_completion(parent); +} + +/** + * prepare_for_tail_block_commit() - Prepare to commit oldest tail blocks. + * + * Implements vdo_action_preamble_fn. + */ +static void prepare_for_tail_block_commit(void *context, struct vdo_completion *parent) +{ + struct slab_depot *depot = context; + + depot->active_release_request = depot->new_release_request; + vdo_finish_completion(parent); +} + +/** + * schedule_tail_block_commit() - Schedule a tail block commit if necessary. + * + * This method should not be called directly. Rather, call vdo_schedule_default_action() on the + * depot's action manager. + * + * Implements vdo_action_scheduler_fn. + */ +static bool schedule_tail_block_commit(void *context) +{ + struct slab_depot *depot = context; + + if (depot->new_release_request == depot->active_release_request) + return false; + + return vdo_schedule_action(depot->action_manager, + prepare_for_tail_block_commit, + release_tail_block_locks, + NULL, NULL); +} + +/** + * initialize_slab_scrubber() - Initialize an allocator's slab scrubber. + * @allocator: The allocator being initialized + * + * Return: VDO_SUCCESS or an error. + */ +static int initialize_slab_scrubber(struct block_allocator *allocator) +{ + struct slab_scrubber *scrubber = &allocator->scrubber; + block_count_t slab_journal_size = + allocator->depot->slab_config.slab_journal_blocks; + char *journal_data; + int result; + + result = vdo_allocate(VDO_BLOCK_SIZE * slab_journal_size, + char, __func__, &journal_data); + if (result != VDO_SUCCESS) + return result; + + result = allocate_vio_components(allocator->completion.vdo, + VIO_TYPE_SLAB_JOURNAL, + VIO_PRIORITY_METADATA, + allocator, slab_journal_size, + journal_data, &scrubber->vio); + if (result != VDO_SUCCESS) { + vdo_free(journal_data); + return result; + } + + INIT_LIST_HEAD(&scrubber->high_priority_slabs); + INIT_LIST_HEAD(&scrubber->slabs); + vdo_set_admin_state_code(&scrubber->admin_state, VDO_ADMIN_STATE_SUSPENDED); + return VDO_SUCCESS; +} + +/** + * initialize_slab_summary_block() - Initialize a slab_summary_block. + * @allocator: The allocator which owns the block. + * @index: The index of this block in its zone's summary. + * + * Return: VDO_SUCCESS or an error. + */ +static int __must_check initialize_slab_summary_block(struct block_allocator *allocator, + block_count_t index) +{ + struct slab_summary_block *block = &allocator->summary_blocks[index]; + int result; + + result = vdo_allocate(VDO_BLOCK_SIZE, char, __func__, &block->outgoing_entries); + if (result != VDO_SUCCESS) + return result; + + result = allocate_vio_components(allocator->depot->vdo, VIO_TYPE_SLAB_SUMMARY, + VIO_PRIORITY_METADATA, NULL, 1, + block->outgoing_entries, &block->vio); + if (result != VDO_SUCCESS) + return result; + + block->allocator = allocator; + block->entries = &allocator->summary_entries[VDO_SLAB_SUMMARY_ENTRIES_PER_BLOCK * index]; + block->index = index; + return VDO_SUCCESS; +} + +static int __must_check initialize_block_allocator(struct slab_depot *depot, + zone_count_t zone) +{ + int result; + block_count_t i; + struct block_allocator *allocator = &depot->allocators[zone]; + struct vdo *vdo = depot->vdo; + block_count_t max_free_blocks = depot->slab_config.data_blocks; + unsigned int max_priority = (2 + ilog2(max_free_blocks)); + + *allocator = (struct block_allocator) { + .depot = depot, + .zone_number = zone, + .thread_id = vdo->thread_config.physical_threads[zone], + .nonce = vdo->states.vdo.nonce, + }; + + INIT_LIST_HEAD(&allocator->dirty_slab_journals); + vdo_set_admin_state_code(&allocator->state, VDO_ADMIN_STATE_NORMAL_OPERATION); + result = vdo_register_read_only_listener(vdo, allocator, + notify_block_allocator_of_read_only_mode, + allocator->thread_id); + if (result != VDO_SUCCESS) + return result; + + vdo_initialize_completion(&allocator->completion, vdo, VDO_BLOCK_ALLOCATOR_COMPLETION); + result = make_vio_pool(vdo, BLOCK_ALLOCATOR_VIO_POOL_SIZE, allocator->thread_id, + VIO_TYPE_SLAB_JOURNAL, VIO_PRIORITY_METADATA, + allocator, &allocator->vio_pool); + if (result != VDO_SUCCESS) + return result; + + result = initialize_slab_scrubber(allocator); + if (result != VDO_SUCCESS) + return result; + + result = vdo_make_priority_table(max_priority, &allocator->prioritized_slabs); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate(VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE, + struct slab_summary_block, __func__, + &allocator->summary_blocks); + if (result != VDO_SUCCESS) + return result; + + vdo_set_admin_state_code(&allocator->summary_state, + VDO_ADMIN_STATE_NORMAL_OPERATION); + allocator->summary_entries = depot->summary_entries + (MAX_VDO_SLABS * zone); + + /* Initialize each summary block. */ + for (i = 0; i < VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE; i++) { + result = initialize_slab_summary_block(allocator, i); + if (result != VDO_SUCCESS) + return result; + } + + /* + * Performing well atop thin provisioned storage requires either that VDO discards freed + * blocks, or that the block allocator try to use slabs that already have allocated blocks + * in preference to slabs that have never been opened. For reasons we have not been able to + * fully understand, some SSD machines have been have been very sensitive (50% reduction in + * test throughput) to very slight differences in the timing and locality of block + * allocation. Assigning a low priority to unopened slabs (max_priority/2, say) would be + * ideal for the story, but anything less than a very high threshold (max_priority - 1) + * hurts on these machines. + * + * This sets the free block threshold for preferring to open an unopened slab to the binary + * floor of 3/4ths the total number of data blocks in a slab, which will generally evaluate + * to about half the slab size. + */ + allocator->unopened_slab_priority = (1 + ilog2((max_free_blocks * 3) / 4)); + + return VDO_SUCCESS; +} + +static int allocate_components(struct slab_depot *depot, + struct partition *summary_partition) +{ + int result; + zone_count_t zone; + slab_count_t slab_count; + u8 hint; + u32 i; + const struct thread_config *thread_config = &depot->vdo->thread_config; + + result = vdo_make_action_manager(depot->zone_count, get_allocator_thread_id, + thread_config->journal_thread, depot, + schedule_tail_block_commit, + depot->vdo, &depot->action_manager); + if (result != VDO_SUCCESS) + return result; + + depot->origin = depot->first_block; + + /* block size must be a multiple of entry size */ + BUILD_BUG_ON((VDO_BLOCK_SIZE % sizeof(struct slab_summary_entry)) != 0); + + depot->summary_origin = summary_partition->offset; + depot->hint_shift = vdo_get_slab_summary_hint_shift(depot->slab_size_shift); + result = vdo_allocate(MAXIMUM_VDO_SLAB_SUMMARY_ENTRIES, + struct slab_summary_entry, __func__, + &depot->summary_entries); + if (result != VDO_SUCCESS) + return result; + + + /* Initialize all the entries. */ + hint = compute_fullness_hint(depot, depot->slab_config.data_blocks); + for (i = 0; i < MAXIMUM_VDO_SLAB_SUMMARY_ENTRIES; i++) { + /* + * This default tail block offset must be reflected in + * slabJournal.c::read_slab_journal_tail(). + */ + depot->summary_entries[i] = (struct slab_summary_entry) { + .tail_block_offset = 0, + .fullness_hint = hint, + .load_ref_counts = false, + .is_dirty = false, + }; + } + + slab_count = vdo_compute_slab_count(depot->first_block, depot->last_block, + depot->slab_size_shift); + if (thread_config->physical_zone_count > slab_count) { + return vdo_log_error_strerror(VDO_BAD_CONFIGURATION, + "%u physical zones exceeds slab count %u", + thread_config->physical_zone_count, + slab_count); + } + + /* Initialize the block allocators. */ + for (zone = 0; zone < depot->zone_count; zone++) { + result = initialize_block_allocator(depot, zone); + if (result != VDO_SUCCESS) + return result; + } + + /* Allocate slabs. */ + result = allocate_slabs(depot, slab_count); + if (result != VDO_SUCCESS) + return result; + + /* Use the new slabs. */ + for (i = depot->slab_count; i < depot->new_slab_count; i++) { + struct vdo_slab *slab = depot->new_slabs[i]; + + register_slab_with_allocator(slab->allocator, slab); + WRITE_ONCE(depot->slab_count, depot->slab_count + 1); + } + + depot->slabs = depot->new_slabs; + depot->new_slabs = NULL; + depot->new_slab_count = 0; + + return VDO_SUCCESS; +} + +/** + * vdo_decode_slab_depot() - Make a slab depot and configure it with the state read from the super + * block. + * @state: The slab depot state from the super block. + * @vdo: The VDO which will own the depot. + * @summary_partition: The partition which holds the slab summary. + * @depot_ptr: A pointer to hold the depot. + * + * Return: A success or error code. + */ +int vdo_decode_slab_depot(struct slab_depot_state_2_0 state, struct vdo *vdo, + struct partition *summary_partition, + struct slab_depot **depot_ptr) +{ + unsigned int slab_size_shift; + struct slab_depot *depot; + int result; + + /* + * Calculate the bit shift for efficiently mapping block numbers to slabs. Using a shift + * requires that the slab size be a power of two. + */ + block_count_t slab_size = state.slab_config.slab_blocks; + + if (!is_power_of_2(slab_size)) { + return vdo_log_error_strerror(UDS_INVALID_ARGUMENT, + "slab size must be a power of two"); + } + slab_size_shift = ilog2(slab_size); + + result = vdo_allocate_extended(struct slab_depot, + vdo->thread_config.physical_zone_count, + struct block_allocator, __func__, &depot); + if (result != VDO_SUCCESS) + return result; + + depot->vdo = vdo; + depot->old_zone_count = state.zone_count; + depot->zone_count = vdo->thread_config.physical_zone_count; + depot->slab_config = state.slab_config; + depot->first_block = state.first_block; + depot->last_block = state.last_block; + depot->slab_size_shift = slab_size_shift; + + result = allocate_components(depot, summary_partition); + if (result != VDO_SUCCESS) { + vdo_free_slab_depot(depot); + return result; + } + + *depot_ptr = depot; + return VDO_SUCCESS; +} + +static void uninitialize_allocator_summary(struct block_allocator *allocator) +{ + block_count_t i; + + if (allocator->summary_blocks == NULL) + return; + + for (i = 0; i < VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE; i++) { + free_vio_components(&allocator->summary_blocks[i].vio); + vdo_free(vdo_forget(allocator->summary_blocks[i].outgoing_entries)); + } + + vdo_free(vdo_forget(allocator->summary_blocks)); +} + +/** + * vdo_free_slab_depot() - Destroy a slab depot. + * @depot: The depot to destroy. + */ +void vdo_free_slab_depot(struct slab_depot *depot) +{ + zone_count_t zone = 0; + + if (depot == NULL) + return; + + vdo_abandon_new_slabs(depot); + + for (zone = 0; zone < depot->zone_count; zone++) { + struct block_allocator *allocator = &depot->allocators[zone]; + + if (allocator->eraser != NULL) + dm_kcopyd_client_destroy(vdo_forget(allocator->eraser)); + + uninitialize_allocator_summary(allocator); + uninitialize_scrubber_vio(&allocator->scrubber); + free_vio_pool(vdo_forget(allocator->vio_pool)); + vdo_free_priority_table(vdo_forget(allocator->prioritized_slabs)); + } + + if (depot->slabs != NULL) { + slab_count_t i; + + for (i = 0; i < depot->slab_count; i++) + free_slab(vdo_forget(depot->slabs[i])); + } + + vdo_free(vdo_forget(depot->slabs)); + vdo_free(vdo_forget(depot->action_manager)); + vdo_free(vdo_forget(depot->summary_entries)); + vdo_free(depot); +} + +/** + * vdo_record_slab_depot() - Record the state of a slab depot for encoding into the super block. + * @depot: The depot to encode. + * + * Return: The depot state. + */ +struct slab_depot_state_2_0 vdo_record_slab_depot(const struct slab_depot *depot) +{ + /* + * If this depot is currently using 0 zones, it must have been synchronously loaded by a + * tool and is now being saved. We did not load and combine the slab summary, so we still + * need to do that next time we load with the old zone count rather than 0. + */ + struct slab_depot_state_2_0 state; + zone_count_t zones_to_record = depot->zone_count; + + if (depot->zone_count == 0) + zones_to_record = depot->old_zone_count; + + state = (struct slab_depot_state_2_0) { + .slab_config = depot->slab_config, + .first_block = depot->first_block, + .last_block = depot->last_block, + .zone_count = zones_to_record, + }; + + return state; +} + +/** + * vdo_allocate_reference_counters() - Allocate the reference counters for all slabs in the depot. + * + * Context: This method may be called only before entering normal operation from the load thread. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_allocate_reference_counters(struct slab_depot *depot) +{ + struct slab_iterator iterator = + get_depot_slab_iterator(depot, depot->slab_count - 1, 0, 1); + + while (iterator.next != NULL) { + int result = allocate_slab_counters(next_slab(&iterator)); + + if (result != VDO_SUCCESS) + return result; + } + + return VDO_SUCCESS; +} + +/** + * get_slab_number() - Get the number of the slab that contains a specified block. + * @depot: The slab depot. + * @pbn: The physical block number. + * @slab_number_ptr: A pointer to hold the slab number. + * + * Return: VDO_SUCCESS or an error. + */ +static int __must_check get_slab_number(const struct slab_depot *depot, + physical_block_number_t pbn, + slab_count_t *slab_number_ptr) +{ + slab_count_t slab_number; + + if (pbn < depot->first_block) + return VDO_OUT_OF_RANGE; + + slab_number = (pbn - depot->first_block) >> depot->slab_size_shift; + if (slab_number >= depot->slab_count) + return VDO_OUT_OF_RANGE; + + *slab_number_ptr = slab_number; + return VDO_SUCCESS; +} + +/** + * vdo_get_slab() - Get the slab object for the slab that contains a specified block. + * @depot: The slab depot. + * @pbn: The physical block number. + * + * Will put the VDO in read-only mode if the PBN is not a valid data block nor the zero block. + * + * Return: The slab containing the block, or NULL if the block number is the zero block or + * otherwise out of range. + */ +struct vdo_slab *vdo_get_slab(const struct slab_depot *depot, + physical_block_number_t pbn) +{ + slab_count_t slab_number; + int result; + + if (pbn == VDO_ZERO_BLOCK) + return NULL; + + result = get_slab_number(depot, pbn, &slab_number); + if (result != VDO_SUCCESS) { + vdo_enter_read_only_mode(depot->vdo, result); + return NULL; + } + + return depot->slabs[slab_number]; +} + +/** + * vdo_get_increment_limit() - Determine how many new references a block can acquire. + * @depot: The slab depot. + * @pbn: The physical block number that is being queried. + * + * Context: This method must be called from the physical zone thread of the PBN. + * + * Return: The number of available references. + */ +u8 vdo_get_increment_limit(struct slab_depot *depot, physical_block_number_t pbn) +{ + struct vdo_slab *slab = vdo_get_slab(depot, pbn); + vdo_refcount_t *counter_ptr = NULL; + int result; + + if ((slab == NULL) || (slab->status != VDO_SLAB_REBUILT)) + return 0; + + result = get_reference_counter(slab, pbn, &counter_ptr); + if (result != VDO_SUCCESS) + return 0; + + if (*counter_ptr == PROVISIONAL_REFERENCE_COUNT) + return (MAXIMUM_REFERENCE_COUNT - 1); + + return (MAXIMUM_REFERENCE_COUNT - *counter_ptr); +} + +/** + * vdo_is_physical_data_block() - Determine whether the given PBN refers to a data block. + * @depot: The depot. + * @pbn: The physical block number to ask about. + * + * Return: True if the PBN corresponds to a data block. + */ +bool vdo_is_physical_data_block(const struct slab_depot *depot, + physical_block_number_t pbn) +{ + slab_count_t slab_number; + slab_block_number sbn; + + return ((pbn == VDO_ZERO_BLOCK) || + ((get_slab_number(depot, pbn, &slab_number) == VDO_SUCCESS) && + (slab_block_number_from_pbn(depot->slabs[slab_number], pbn, &sbn) == + VDO_SUCCESS))); +} + +/** + * vdo_get_slab_depot_allocated_blocks() - Get the total number of data blocks allocated across all + * the slabs in the depot. + * @depot: The slab depot. + * + * This is the total number of blocks with a non-zero reference count. + * + * Context: This may be called from any thread. + * + * Return: The total number of blocks with a non-zero reference count. + */ +block_count_t vdo_get_slab_depot_allocated_blocks(const struct slab_depot *depot) +{ + block_count_t total = 0; + zone_count_t zone; + + for (zone = 0; zone < depot->zone_count; zone++) { + /* The allocators are responsible for thread safety. */ + total += READ_ONCE(depot->allocators[zone].allocated_blocks); + } + + return total; +} + +/** + * vdo_get_slab_depot_data_blocks() - Get the total number of data blocks in all the slabs in the + * depot. + * @depot: The slab depot. + * + * Context: This may be called from any thread. + * + * Return: The total number of data blocks in all slabs. + */ +block_count_t vdo_get_slab_depot_data_blocks(const struct slab_depot *depot) +{ + return (READ_ONCE(depot->slab_count) * depot->slab_config.data_blocks); +} + +/** + * finish_combining_zones() - Clean up after saving out the combined slab summary. + * @completion: The vio which was used to write the summary data. + */ +static void finish_combining_zones(struct vdo_completion *completion) +{ + int result = completion->result; + struct vdo_completion *parent = completion->parent; + + free_vio(as_vio(vdo_forget(completion))); + vdo_fail_completion(parent, result); +} + +static void handle_combining_error(struct vdo_completion *completion) +{ + vio_record_metadata_io_error(as_vio(completion)); + finish_combining_zones(completion); +} + +static void write_summary_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct vdo *vdo = vio->completion.vdo; + + continue_vio_after_io(vio, finish_combining_zones, + vdo->thread_config.admin_thread); +} + +/** + * combine_summaries() - Treating the current entries buffer as the on-disk value of all zones, + * update every zone to the correct values for every slab. + * @depot: The depot whose summary entries should be combined. + */ +static void combine_summaries(struct slab_depot *depot) +{ + /* + * Combine all the old summary data into the portion of the buffer corresponding to the + * first zone. + */ + zone_count_t zone = 0; + struct slab_summary_entry *entries = depot->summary_entries; + + if (depot->old_zone_count > 1) { + slab_count_t entry_number; + + for (entry_number = 0; entry_number < MAX_VDO_SLABS; entry_number++) { + if (zone != 0) { + memcpy(entries + entry_number, + entries + (zone * MAX_VDO_SLABS) + entry_number, + sizeof(struct slab_summary_entry)); + } + + zone++; + if (zone == depot->old_zone_count) + zone = 0; + } + } + + /* Copy the combined data to each zones's region of the buffer. */ + for (zone = 1; zone < MAX_VDO_PHYSICAL_ZONES; zone++) { + memcpy(entries + (zone * MAX_VDO_SLABS), entries, + MAX_VDO_SLABS * sizeof(struct slab_summary_entry)); + } +} + +/** + * finish_loading_summary() - Finish loading slab summary data. + * @completion: The vio which was used to read the summary data. + * + * Combines the slab summary data from all the previously written zones and copies the combined + * summary to each partition's data region. Then writes the combined summary back out to disk. This + * callback is registered in load_summary_endio(). + */ +static void finish_loading_summary(struct vdo_completion *completion) +{ + struct slab_depot *depot = completion->vdo->depot; + + /* Combine the summary from each zone so each zone is correct for all slabs. */ + combine_summaries(depot); + + /* Write the combined summary back out. */ + vdo_submit_metadata_vio(as_vio(completion), depot->summary_origin, + write_summary_endio, handle_combining_error, + REQ_OP_WRITE); +} + +static void load_summary_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct vdo *vdo = vio->completion.vdo; + + continue_vio_after_io(vio, finish_loading_summary, + vdo->thread_config.admin_thread); +} + +/** + * load_slab_summary() - The preamble of a load operation. + * + * Implements vdo_action_preamble_fn. + */ +static void load_slab_summary(void *context, struct vdo_completion *parent) +{ + int result; + struct vio *vio; + struct slab_depot *depot = context; + const struct admin_state_code *operation = + vdo_get_current_manager_operation(depot->action_manager); + + result = create_multi_block_metadata_vio(depot->vdo, VIO_TYPE_SLAB_SUMMARY, + VIO_PRIORITY_METADATA, parent, + VDO_SLAB_SUMMARY_BLOCKS, + (char *) depot->summary_entries, &vio); + if (result != VDO_SUCCESS) { + vdo_fail_completion(parent, result); + return; + } + + if ((operation == VDO_ADMIN_STATE_FORMATTING) || + (operation == VDO_ADMIN_STATE_LOADING_FOR_REBUILD)) { + finish_loading_summary(&vio->completion); + return; + } + + vdo_submit_metadata_vio(vio, depot->summary_origin, load_summary_endio, + handle_combining_error, REQ_OP_READ); +} + +/* Implements vdo_zone_action_fn. */ +static void load_allocator(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct slab_depot *depot = context; + + vdo_start_loading(&depot->allocators[zone_number].state, + vdo_get_current_manager_operation(depot->action_manager), + parent, initiate_load); +} + +/** + * vdo_load_slab_depot() - Asynchronously load any slab depot state that isn't included in the + * super_block component. + * @depot: The depot to load. + * @operation: The type of load to perform. + * @parent: The completion to notify when the load is complete. + * @context: Additional context for the load operation; may be NULL. + * + * This method may be called only before entering normal operation from the load thread. + */ +void vdo_load_slab_depot(struct slab_depot *depot, + const struct admin_state_code *operation, + struct vdo_completion *parent, void *context) +{ + if (!vdo_assert_load_operation(operation, parent)) + return; + + vdo_schedule_operation_with_context(depot->action_manager, operation, + load_slab_summary, load_allocator, + NULL, context, parent); +} + +/* Implements vdo_zone_action_fn. */ +static void prepare_to_allocate(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct slab_depot *depot = context; + struct block_allocator *allocator = &depot->allocators[zone_number]; + int result; + + result = vdo_prepare_slabs_for_allocation(allocator); + if (result != VDO_SUCCESS) { + vdo_fail_completion(parent, result); + return; + } + + scrub_slabs(allocator, parent); +} + +/** + * vdo_prepare_slab_depot_to_allocate() - Prepare the slab depot to come online and start + * allocating blocks. + * @depot: The depot to prepare. + * @load_type: The load type. + * @parent: The completion to notify when the operation is complete. + * + * This method may be called only before entering normal operation from the load thread. It must be + * called before allocation may proceed. + */ +void vdo_prepare_slab_depot_to_allocate(struct slab_depot *depot, + enum slab_depot_load_type load_type, + struct vdo_completion *parent) +{ + depot->load_type = load_type; + atomic_set(&depot->zones_to_scrub, depot->zone_count); + vdo_schedule_action(depot->action_manager, NULL, + prepare_to_allocate, NULL, parent); +} + +/** + * vdo_update_slab_depot_size() - Update the slab depot to reflect its new size in memory. + * @depot: The depot to update. + * + * This size is saved to disk as part of the super block. + */ +void vdo_update_slab_depot_size(struct slab_depot *depot) +{ + depot->last_block = depot->new_last_block; +} + +/** + * vdo_prepare_to_grow_slab_depot() - Allocate new memory needed for a resize of a slab depot to + * the given size. + * @depot: The depot to prepare to resize. + * @partition: The new depot partition + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_prepare_to_grow_slab_depot(struct slab_depot *depot, + const struct partition *partition) +{ + struct slab_depot_state_2_0 new_state; + int result; + slab_count_t new_slab_count; + + if ((partition->count >> depot->slab_size_shift) <= depot->slab_count) + return VDO_INCREMENT_TOO_SMALL; + + /* Generate the depot configuration for the new block count. */ + VDO_ASSERT_LOG_ONLY(depot->first_block == partition->offset, + "New slab depot partition doesn't change origin"); + result = vdo_configure_slab_depot(partition, depot->slab_config, + depot->zone_count, &new_state); + if (result != VDO_SUCCESS) + return result; + + new_slab_count = vdo_compute_slab_count(depot->first_block, + new_state.last_block, + depot->slab_size_shift); + if (new_slab_count <= depot->slab_count) + return vdo_log_error_strerror(VDO_INCREMENT_TOO_SMALL, + "Depot can only grow"); + if (new_slab_count == depot->new_slab_count) { + /* Check it out, we've already got all the new slabs allocated! */ + return VDO_SUCCESS; + } + + vdo_abandon_new_slabs(depot); + result = allocate_slabs(depot, new_slab_count); + if (result != VDO_SUCCESS) { + vdo_abandon_new_slabs(depot); + return result; + } + + depot->new_size = partition->count; + depot->old_last_block = depot->last_block; + depot->new_last_block = new_state.last_block; + + return VDO_SUCCESS; +} + +/** + * finish_registration() - Finish registering new slabs now that all of the allocators have + * received their new slabs. + * + * Implements vdo_action_conclusion_fn. + */ +static int finish_registration(void *context) +{ + struct slab_depot *depot = context; + + WRITE_ONCE(depot->slab_count, depot->new_slab_count); + vdo_free(depot->slabs); + depot->slabs = depot->new_slabs; + depot->new_slabs = NULL; + depot->new_slab_count = 0; + return VDO_SUCCESS; +} + +/* Implements vdo_zone_action_fn. */ +static void register_new_slabs(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct slab_depot *depot = context; + struct block_allocator *allocator = &depot->allocators[zone_number]; + slab_count_t i; + + for (i = depot->slab_count; i < depot->new_slab_count; i++) { + struct vdo_slab *slab = depot->new_slabs[i]; + + if (slab->allocator == allocator) + register_slab_with_allocator(allocator, slab); + } + + vdo_finish_completion(parent); +} + +/** + * vdo_use_new_slabs() - Use the new slabs allocated for resize. + * @depot: The depot. + * @parent: The object to notify when complete. + */ +void vdo_use_new_slabs(struct slab_depot *depot, struct vdo_completion *parent) +{ + VDO_ASSERT_LOG_ONLY(depot->new_slabs != NULL, "Must have new slabs to use"); + vdo_schedule_operation(depot->action_manager, + VDO_ADMIN_STATE_SUSPENDED_OPERATION, + NULL, register_new_slabs, + finish_registration, parent); +} + +/** + * stop_scrubbing() - Tell the scrubber to stop scrubbing after it finishes the slab it is + * currently working on. + * @scrubber: The scrubber to stop. + * @parent: The completion to notify when scrubbing has stopped. + */ +static void stop_scrubbing(struct block_allocator *allocator) +{ + struct slab_scrubber *scrubber = &allocator->scrubber; + + if (vdo_is_state_quiescent(&scrubber->admin_state)) { + vdo_finish_completion(&allocator->completion); + } else { + vdo_start_draining(&scrubber->admin_state, + VDO_ADMIN_STATE_SUSPENDING, + &allocator->completion, NULL); + } +} + +/* Implements vdo_admin_initiator_fn. */ +static void initiate_summary_drain(struct admin_state *state) +{ + check_summary_drain_complete(container_of(state, struct block_allocator, + summary_state)); +} + +static void do_drain_step(struct vdo_completion *completion) +{ + struct block_allocator *allocator = vdo_as_block_allocator(completion); + + vdo_prepare_completion_for_requeue(&allocator->completion, do_drain_step, + handle_operation_error, allocator->thread_id, + NULL); + switch (++allocator->drain_step) { + case VDO_DRAIN_ALLOCATOR_STEP_SCRUBBER: + stop_scrubbing(allocator); + return; + + case VDO_DRAIN_ALLOCATOR_STEP_SLABS: + apply_to_slabs(allocator, do_drain_step); + return; + + case VDO_DRAIN_ALLOCATOR_STEP_SUMMARY: + vdo_start_draining(&allocator->summary_state, + vdo_get_admin_state_code(&allocator->state), + completion, initiate_summary_drain); + return; + + case VDO_DRAIN_ALLOCATOR_STEP_FINISHED: + VDO_ASSERT_LOG_ONLY(!is_vio_pool_busy(allocator->vio_pool), + "vio pool not busy"); + vdo_finish_draining_with_result(&allocator->state, completion->result); + return; + + default: + vdo_finish_draining_with_result(&allocator->state, UDS_BAD_STATE); + } +} + +/* Implements vdo_admin_initiator_fn. */ +static void initiate_drain(struct admin_state *state) +{ + struct block_allocator *allocator = + container_of(state, struct block_allocator, state); + + allocator->drain_step = VDO_DRAIN_ALLOCATOR_START; + do_drain_step(&allocator->completion); +} + +/* + * Drain all allocator I/O. Depending upon the type of drain, some or all dirty metadata may be + * written to disk. The type of drain will be determined from the state of the allocator's depot. + * + * Implements vdo_zone_action_fn. + */ +static void drain_allocator(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct slab_depot *depot = context; + + vdo_start_draining(&depot->allocators[zone_number].state, + vdo_get_current_manager_operation(depot->action_manager), + parent, initiate_drain); +} + +/** + * vdo_drain_slab_depot() - Drain all slab depot I/O. + * @depot: The depot to drain. + * @operation: The drain operation (flush, rebuild, suspend, or save). + * @parent: The completion to finish when the drain is complete. + * + * If saving, or flushing, all dirty depot metadata will be written out. If saving or suspending, + * the depot will be left in a suspended state. + */ +void vdo_drain_slab_depot(struct slab_depot *depot, + const struct admin_state_code *operation, + struct vdo_completion *parent) +{ + vdo_schedule_operation(depot->action_manager, operation, + NULL, drain_allocator, NULL, parent); +} + +/** + * resume_scrubbing() - Tell the scrubber to resume scrubbing if it has been stopped. + * @allocator: The allocator being resumed. + */ +static void resume_scrubbing(struct block_allocator *allocator) +{ + int result; + struct slab_scrubber *scrubber = &allocator->scrubber; + + if (!has_slabs_to_scrub(scrubber)) { + vdo_finish_completion(&allocator->completion); + return; + } + + result = vdo_resume_if_quiescent(&scrubber->admin_state); + if (result != VDO_SUCCESS) { + vdo_fail_completion(&allocator->completion, result); + return; + } + + scrub_next_slab(scrubber); + vdo_finish_completion(&allocator->completion); +} + +static void do_resume_step(struct vdo_completion *completion) +{ + struct block_allocator *allocator = vdo_as_block_allocator(completion); + + vdo_prepare_completion_for_requeue(&allocator->completion, do_resume_step, + handle_operation_error, + allocator->thread_id, NULL); + switch (--allocator->drain_step) { + case VDO_DRAIN_ALLOCATOR_STEP_SUMMARY: + vdo_fail_completion(completion, + vdo_resume_if_quiescent(&allocator->summary_state)); + return; + + case VDO_DRAIN_ALLOCATOR_STEP_SLABS: + apply_to_slabs(allocator, do_resume_step); + return; + + case VDO_DRAIN_ALLOCATOR_STEP_SCRUBBER: + resume_scrubbing(allocator); + return; + + case VDO_DRAIN_ALLOCATOR_START: + vdo_finish_resuming_with_result(&allocator->state, completion->result); + return; + + default: + vdo_finish_resuming_with_result(&allocator->state, UDS_BAD_STATE); + } +} + +/* Implements vdo_admin_initiator_fn. */ +static void initiate_resume(struct admin_state *state) +{ + struct block_allocator *allocator = + container_of(state, struct block_allocator, state); + + allocator->drain_step = VDO_DRAIN_ALLOCATOR_STEP_FINISHED; + do_resume_step(&allocator->completion); +} + +/* Implements vdo_zone_action_fn. */ +static void resume_allocator(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct slab_depot *depot = context; + + vdo_start_resuming(&depot->allocators[zone_number].state, + vdo_get_current_manager_operation(depot->action_manager), + parent, initiate_resume); +} + +/** + * vdo_resume_slab_depot() - Resume a suspended slab depot. + * @depot: The depot to resume. + * @parent: The completion to finish when the depot has resumed. + */ +void vdo_resume_slab_depot(struct slab_depot *depot, struct vdo_completion *parent) +{ + if (vdo_is_read_only(depot->vdo)) { + vdo_continue_completion(parent, VDO_READ_ONLY); + return; + } + + vdo_schedule_operation(depot->action_manager, VDO_ADMIN_STATE_RESUMING, + NULL, resume_allocator, NULL, parent); +} + +/** + * vdo_commit_oldest_slab_journal_tail_blocks() - Commit all dirty tail blocks which are locking a + * given recovery journal block. + * @depot: The depot. + * @recovery_block_number: The sequence number of the recovery journal block whose locks should be + * released. + * + * Context: This method must be called from the journal zone thread. + */ +void vdo_commit_oldest_slab_journal_tail_blocks(struct slab_depot *depot, + sequence_number_t recovery_block_number) +{ + if (depot == NULL) + return; + + depot->new_release_request = recovery_block_number; + vdo_schedule_default_action(depot->action_manager); +} + +/* Implements vdo_zone_action_fn. */ +static void scrub_all_unrecovered_slabs(void *context, zone_count_t zone_number, + struct vdo_completion *parent) +{ + struct slab_depot *depot = context; + + scrub_slabs(&depot->allocators[zone_number], NULL); + vdo_launch_completion(parent); +} + +/** + * vdo_scrub_all_unrecovered_slabs() - Scrub all unrecovered slabs. + * @depot: The depot to scrub. + * @parent: The object to notify when scrubbing has been launched for all zones. + */ +void vdo_scrub_all_unrecovered_slabs(struct slab_depot *depot, + struct vdo_completion *parent) +{ + vdo_schedule_action(depot->action_manager, NULL, + scrub_all_unrecovered_slabs, + NULL, parent); +} + +/** + * get_block_allocator_statistics() - Get the total of the statistics from all the block allocators + * in the depot. + * @depot: The slab depot. + * + * Return: The statistics from all block allocators in the depot. + */ +static struct block_allocator_statistics __must_check +get_block_allocator_statistics(const struct slab_depot *depot) +{ + struct block_allocator_statistics totals; + zone_count_t zone; + + memset(&totals, 0, sizeof(totals)); + + for (zone = 0; zone < depot->zone_count; zone++) { + const struct block_allocator *allocator = &depot->allocators[zone]; + const struct block_allocator_statistics *stats = &allocator->statistics; + + totals.slab_count += allocator->slab_count; + totals.slabs_opened += READ_ONCE(stats->slabs_opened); + totals.slabs_reopened += READ_ONCE(stats->slabs_reopened); + } + + return totals; +} + +/** + * get_ref_counts_statistics() - Get the cumulative ref_counts statistics for the depot. + * @depot: The slab depot. + * + * Return: The cumulative statistics for all ref_counts in the depot. + */ +static struct ref_counts_statistics __must_check +get_ref_counts_statistics(const struct slab_depot *depot) +{ + struct ref_counts_statistics totals; + zone_count_t zone; + + memset(&totals, 0, sizeof(totals)); + + for (zone = 0; zone < depot->zone_count; zone++) { + totals.blocks_written += + READ_ONCE(depot->allocators[zone].ref_counts_statistics.blocks_written); + } + + return totals; +} + +/** + * get_slab_journal_statistics() - Get the aggregated slab journal statistics for the depot. + * @depot: The slab depot. + * + * Return: The aggregated statistics for all slab journals in the depot. + */ +static struct slab_journal_statistics __must_check +get_slab_journal_statistics(const struct slab_depot *depot) +{ + struct slab_journal_statistics totals; + zone_count_t zone; + + memset(&totals, 0, sizeof(totals)); + + for (zone = 0; zone < depot->zone_count; zone++) { + const struct slab_journal_statistics *stats = + &depot->allocators[zone].slab_journal_statistics; + + totals.disk_full_count += READ_ONCE(stats->disk_full_count); + totals.flush_count += READ_ONCE(stats->flush_count); + totals.blocked_count += READ_ONCE(stats->blocked_count); + totals.blocks_written += READ_ONCE(stats->blocks_written); + totals.tail_busy_count += READ_ONCE(stats->tail_busy_count); + } + + return totals; +} + +/** + * vdo_get_slab_depot_statistics() - Get all the vdo_statistics fields that are properties of the + * slab depot. + * @depot: The slab depot. + * @stats: The vdo statistics structure to partially fill. + */ +void vdo_get_slab_depot_statistics(const struct slab_depot *depot, + struct vdo_statistics *stats) +{ + slab_count_t slab_count = READ_ONCE(depot->slab_count); + slab_count_t unrecovered = 0; + zone_count_t zone; + + for (zone = 0; zone < depot->zone_count; zone++) { + /* The allocators are responsible for thread safety. */ + unrecovered += READ_ONCE(depot->allocators[zone].scrubber.slab_count); + } + + stats->recovery_percentage = (slab_count - unrecovered) * 100 / slab_count; + stats->allocator = get_block_allocator_statistics(depot); + stats->ref_counts = get_ref_counts_statistics(depot); + stats->slab_journal = get_slab_journal_statistics(depot); + stats->slab_summary = (struct slab_summary_statistics) { + .blocks_written = atomic64_read(&depot->summary_statistics.blocks_written), + }; +} + +/** + * vdo_dump_slab_depot() - Dump the slab depot, in a thread-unsafe fashion. + * @depot: The slab depot. + */ +void vdo_dump_slab_depot(const struct slab_depot *depot) +{ + vdo_log_info("vdo slab depot"); + vdo_log_info(" zone_count=%u old_zone_count=%u slabCount=%u active_release_request=%llu new_release_request=%llu", + (unsigned int) depot->zone_count, + (unsigned int) depot->old_zone_count, READ_ONCE(depot->slab_count), + (unsigned long long) depot->active_release_request, + (unsigned long long) depot->new_release_request); +} diff --git a/drivers/md/dm-vdo/slab-depot.h b/drivers/md/dm-vdo/slab-depot.h new file mode 100644 index 000000000000..f234853501ca --- /dev/null +++ b/drivers/md/dm-vdo/slab-depot.h @@ -0,0 +1,601 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_SLAB_DEPOT_H +#define VDO_SLAB_DEPOT_H + +#include +#include +#include + +#include "numeric.h" + +#include "admin-state.h" +#include "completion.h" +#include "data-vio.h" +#include "encodings.h" +#include "physical-zone.h" +#include "priority-table.h" +#include "recovery-journal.h" +#include "statistics.h" +#include "types.h" +#include "vio.h" +#include "wait-queue.h" + +/* + * A slab_depot is responsible for managing all of the slabs and block allocators of a VDO. It has + * a single array of slabs in order to eliminate the need for additional math in order to compute + * which physical zone a PBN is in. It also has a block_allocator per zone. + * + * Each physical zone has a single dedicated queue and thread for performing all updates to the + * slabs assigned to that zone. The concurrency guarantees of this single-threaded model allow the + * code to omit more fine-grained locking for the various slab structures. Each physical zone + * maintains a separate copy of the slab summary to remove the need for explicit locking on that + * structure as well. + * + * Load operations must be performed on the admin thread. Normal operations, such as allocations + * and reference count updates, must be performed on the appropriate physical zone thread. Requests + * from the recovery journal to commit slab journal tail blocks must be scheduled from the recovery + * journal thread to run on the appropriate physical zone thread. Save operations must be launched + * from the same admin thread as the original load operation. + */ + +enum { + /* The number of vios in the vio pool is proportional to the throughput of the VDO. */ + BLOCK_ALLOCATOR_VIO_POOL_SIZE = 128, +}; + +/* + * Represents the possible status of a block. + */ +enum reference_status { + RS_FREE, /* this block is free */ + RS_SINGLE, /* this block is singly-referenced */ + RS_SHARED, /* this block is shared */ + RS_PROVISIONAL /* this block is provisionally allocated */ +}; + +struct vdo_slab; + +struct journal_lock { + u16 count; + sequence_number_t recovery_start; +}; + +struct slab_journal { + /* A waiter object for getting a VIO pool entry */ + struct vdo_waiter resource_waiter; + /* A waiter object for updating the slab summary */ + struct vdo_waiter slab_summary_waiter; + /* A waiter object for getting a vio with which to flush */ + struct vdo_waiter flush_waiter; + /* The queue of VIOs waiting to make an entry */ + struct vdo_wait_queue entry_waiters; + /* The parent slab reference of this journal */ + struct vdo_slab *slab; + + /* Whether a tail block commit is pending */ + bool waiting_to_commit; + /* Whether the journal is updating the slab summary */ + bool updating_slab_summary; + /* Whether the journal is adding entries from the entry_waiters queue */ + bool adding_entries; + /* Whether a partial write is in progress */ + bool partial_write_in_progress; + + /* The oldest block in the journal on disk */ + sequence_number_t head; + /* The oldest block in the journal which may not be reaped */ + sequence_number_t unreapable; + /* The end of the half-open interval of the active journal */ + sequence_number_t tail; + /* The next journal block to be committed */ + sequence_number_t next_commit; + /* The tail sequence number that is written in the slab summary */ + sequence_number_t summarized; + /* The tail sequence number that was last summarized in slab summary */ + sequence_number_t last_summarized; + + /* The sequence number of the recovery journal lock */ + sequence_number_t recovery_lock; + + /* + * The number of entries which fit in a single block. Can't use the constant because unit + * tests change this number. + */ + journal_entry_count_t entries_per_block; + /* + * The number of full entries which fit in a single block. Can't use the constant because + * unit tests change this number. + */ + journal_entry_count_t full_entries_per_block; + + /* The recovery journal of the VDO (slab journal holds locks on it) */ + struct recovery_journal *recovery_journal; + + /* The statistics shared by all slab journals in our physical zone */ + struct slab_journal_statistics *events; + /* A list of the VIO pool entries for outstanding journal block writes */ + struct list_head uncommitted_blocks; + + /* + * The current tail block header state. This will be packed into the block just before it + * is written. + */ + struct slab_journal_block_header tail_header; + /* A pointer to a block-sized buffer holding the packed block data */ + struct packed_slab_journal_block *block; + + /* The number of blocks in the on-disk journal */ + block_count_t size; + /* The number of blocks at which to start pushing reference blocks */ + block_count_t flushing_threshold; + /* The number of blocks at which all reference blocks should be writing */ + block_count_t flushing_deadline; + /* The number of blocks at which to wait for reference blocks to write */ + block_count_t blocking_threshold; + /* The number of blocks at which to scrub the slab before coming online */ + block_count_t scrubbing_threshold; + + /* This list entry is for block_allocator to keep a queue of dirty journals */ + struct list_head dirty_entry; + + /* The lock for the oldest unreaped block of the journal */ + struct journal_lock *reap_lock; + /* The locks for each on disk block */ + struct journal_lock *locks; +}; + +/* + * Reference_block structure + * + * Blocks are used as a proxy, permitting saves of partial refcounts. + */ +struct reference_block { + /* This block waits on the ref_counts to tell it to write */ + struct vdo_waiter waiter; + /* The slab to which this reference_block belongs */ + struct vdo_slab *slab; + /* The number of references in this block that represent allocations */ + block_size_t allocated_count; + /* The slab journal block on which this block must hold a lock */ + sequence_number_t slab_journal_lock; + /* The slab journal block which should be released when this block is committed */ + sequence_number_t slab_journal_lock_to_release; + /* The point up to which each sector is accurate on disk */ + struct journal_point commit_points[VDO_SECTORS_PER_BLOCK]; + /* Whether this block has been modified since it was written to disk */ + bool is_dirty; + /* Whether this block is currently writing */ + bool is_writing; +}; + +/* The search_cursor represents the saved position of a free block search. */ +struct search_cursor { + /* The reference block containing the current search index */ + struct reference_block *block; + /* The position at which to start searching for the next free counter */ + slab_block_number index; + /* The position just past the last valid counter in the current block */ + slab_block_number end_index; + + /* A pointer to the first reference block in the slab */ + struct reference_block *first_block; + /* A pointer to the last reference block in the slab */ + struct reference_block *last_block; +}; + +enum slab_rebuild_status { + VDO_SLAB_REBUILT, + VDO_SLAB_REPLAYING, + VDO_SLAB_REQUIRES_SCRUBBING, + VDO_SLAB_REQUIRES_HIGH_PRIORITY_SCRUBBING, + VDO_SLAB_REBUILDING, +}; + +/* + * This is the type declaration for the vdo_slab type. A vdo_slab currently consists of a run of + * 2^23 data blocks, but that will soon change to dedicate a small number of those blocks for + * metadata storage for the reference counts and slab journal for the slab. + * + * A reference count is maintained for each physical block number. The vast majority of blocks have + * a very small reference count (usually 0 or 1). For references less than or equal to MAXIMUM_REFS + * (254) the reference count is stored in counters[pbn]. + */ +struct vdo_slab { + /* A list entry to queue this slab in a block_allocator list */ + struct list_head allocq_entry; + + /* The struct block_allocator that owns this slab */ + struct block_allocator *allocator; + + /* The journal for this slab */ + struct slab_journal journal; + + /* The slab number of this slab */ + slab_count_t slab_number; + /* The offset in the allocator partition of the first block in this slab */ + physical_block_number_t start; + /* The offset of the first block past the end of this slab */ + physical_block_number_t end; + /* The starting translated PBN of the slab journal */ + physical_block_number_t journal_origin; + /* The starting translated PBN of the reference counts */ + physical_block_number_t ref_counts_origin; + + /* The administrative state of the slab */ + struct admin_state state; + /* The status of the slab */ + enum slab_rebuild_status status; + /* Whether the slab was ever queued for scrubbing */ + bool was_queued_for_scrubbing; + + /* The priority at which this slab has been queued for allocation */ + u8 priority; + + /* Fields beyond this point are the reference counts for the data blocks in this slab. */ + /* The size of the counters array */ + u32 block_count; + /* The number of free blocks */ + u32 free_blocks; + /* The array of reference counts */ + vdo_refcount_t *counters; /* use vdo_allocate() to align data ptr */ + + /* The saved block pointer and array indexes for the free block search */ + struct search_cursor search_cursor; + + /* A list of the dirty blocks waiting to be written out */ + struct vdo_wait_queue dirty_blocks; + /* The number of blocks which are currently writing */ + size_t active_count; + + /* A waiter object for updating the slab summary */ + struct vdo_waiter summary_waiter; + + /* The latest slab journal for which there has been a reference count update */ + struct journal_point slab_journal_point; + + /* The number of reference count blocks */ + u32 reference_block_count; + /* reference count block array */ + struct reference_block *reference_blocks; +}; + +enum block_allocator_drain_step { + VDO_DRAIN_ALLOCATOR_START, + VDO_DRAIN_ALLOCATOR_STEP_SCRUBBER, + VDO_DRAIN_ALLOCATOR_STEP_SLABS, + VDO_DRAIN_ALLOCATOR_STEP_SUMMARY, + VDO_DRAIN_ALLOCATOR_STEP_FINISHED, +}; + +struct slab_scrubber { + /* The queue of slabs to scrub first */ + struct list_head high_priority_slabs; + /* The queue of slabs to scrub once there are no high_priority_slabs */ + struct list_head slabs; + /* The queue of VIOs waiting for a slab to be scrubbed */ + struct vdo_wait_queue waiters; + + /* + * The number of slabs that are unrecovered or being scrubbed. This field is modified by + * the physical zone thread, but is queried by other threads. + */ + slab_count_t slab_count; + + /* The administrative state of the scrubber */ + struct admin_state admin_state; + /* Whether to only scrub high-priority slabs */ + bool high_priority_only; + /* The slab currently being scrubbed */ + struct vdo_slab *slab; + /* The vio for loading slab journal blocks */ + struct vio vio; +}; + +/* A sub-structure for applying actions in parallel to all an allocator's slabs. */ +struct slab_actor { + /* The number of slabs performing a slab action */ + slab_count_t slab_action_count; + /* The method to call when a slab action has been completed by all slabs */ + vdo_action_fn callback; +}; + +/* A slab_iterator is a structure for iterating over a set of slabs. */ +struct slab_iterator { + struct vdo_slab **slabs; + struct vdo_slab *next; + slab_count_t end; + slab_count_t stride; +}; + +/* + * The slab_summary provides hints during load and recovery about the state of the slabs in order + * to avoid the need to read the slab journals in their entirety before a VDO can come online. + * + * The information in the summary for each slab includes the rough number of free blocks (which is + * used to prioritize scrubbing), the cleanliness of a slab (so that clean slabs containing free + * space will be used on restart), and the location of the tail block of the slab's journal. + * + * The slab_summary has its own partition at the end of the volume which is sized to allow for a + * complete copy of the summary for each of up to 16 physical zones. + * + * During resize, the slab_summary moves its backing partition and is saved once moved; the + * slab_summary is not permitted to overwrite the previous recovery journal space. + * + * The slab_summary does not have its own version information, but relies on the VDO volume version + * number. + */ + +/* + * A slab status is a very small structure for use in determining the ordering of slabs in the + * scrubbing process. + */ +struct slab_status { + slab_count_t slab_number; + bool is_clean; + u8 emptiness; +}; + +struct slab_summary_block { + /* The block_allocator to which this block belongs */ + struct block_allocator *allocator; + /* The index of this block in its zone's summary */ + block_count_t index; + /* Whether this block has a write outstanding */ + bool writing; + /* Ring of updates waiting on the outstanding write */ + struct vdo_wait_queue current_update_waiters; + /* Ring of updates waiting on the next write */ + struct vdo_wait_queue next_update_waiters; + /* The active slab_summary_entry array for this block */ + struct slab_summary_entry *entries; + /* The vio used to write this block */ + struct vio vio; + /* The packed entries, one block long, backing the vio */ + char *outgoing_entries; +}; + +/* + * The statistics for all the slab summary zones owned by this slab summary. These fields are all + * mutated only by their physical zone threads, but are read by other threads when gathering + * statistics for the entire depot. + */ +struct atomic_slab_summary_statistics { + /* Number of blocks written */ + atomic64_t blocks_written; +}; + +struct block_allocator { + struct vdo_completion completion; + /* The slab depot for this allocator */ + struct slab_depot *depot; + /* The nonce of the VDO */ + nonce_t nonce; + /* The physical zone number of this allocator */ + zone_count_t zone_number; + /* The thread ID for this allocator's physical zone */ + thread_id_t thread_id; + /* The number of slabs in this allocator */ + slab_count_t slab_count; + /* The number of the last slab owned by this allocator */ + slab_count_t last_slab; + /* The reduced priority level used to preserve unopened slabs */ + unsigned int unopened_slab_priority; + /* The state of this allocator */ + struct admin_state state; + /* The actor for applying an action to all slabs */ + struct slab_actor slab_actor; + + /* The slab from which blocks are currently being allocated */ + struct vdo_slab *open_slab; + /* A priority queue containing all slabs available for allocation */ + struct priority_table *prioritized_slabs; + /* The slab scrubber */ + struct slab_scrubber scrubber; + /* What phase of the close operation the allocator is to perform */ + enum block_allocator_drain_step drain_step; + + /* + * These statistics are all mutated only by the physical zone thread, but are read by other + * threads when gathering statistics for the entire depot. + */ + /* + * The count of allocated blocks in this zone. Not in block_allocator_statistics for + * historical reasons. + */ + u64 allocated_blocks; + /* Statistics for this block allocator */ + struct block_allocator_statistics statistics; + /* Cumulative statistics for the slab journals in this zone */ + struct slab_journal_statistics slab_journal_statistics; + /* Cumulative statistics for the reference counters in this zone */ + struct ref_counts_statistics ref_counts_statistics; + + /* + * This is the head of a queue of slab journals which have entries in their tail blocks + * which have not yet started to commit. When the recovery journal is under space pressure, + * slab journals which have uncommitted entries holding a lock on the recovery journal head + * are forced to commit their blocks early. This list is kept in order, with the tail + * containing the slab journal holding the most recent recovery journal lock. + */ + struct list_head dirty_slab_journals; + + /* The vio pool for reading and writing block allocator metadata */ + struct vio_pool *vio_pool; + /* The dm_kcopyd client for erasing slab journals */ + struct dm_kcopyd_client *eraser; + /* Iterator over the slabs to be erased */ + struct slab_iterator slabs_to_erase; + + /* The portion of the slab summary managed by this allocator */ + /* The state of the slab summary */ + struct admin_state summary_state; + /* The number of outstanding summary writes */ + block_count_t summary_write_count; + /* The array (owned by the blocks) of all entries */ + struct slab_summary_entry *summary_entries; + /* The array of slab_summary_blocks */ + struct slab_summary_block *summary_blocks; +}; + +enum slab_depot_load_type { + VDO_SLAB_DEPOT_NORMAL_LOAD, + VDO_SLAB_DEPOT_RECOVERY_LOAD, + VDO_SLAB_DEPOT_REBUILD_LOAD +}; + +struct slab_depot { + zone_count_t zone_count; + zone_count_t old_zone_count; + struct vdo *vdo; + struct slab_config slab_config; + struct action_manager *action_manager; + + physical_block_number_t first_block; + physical_block_number_t last_block; + physical_block_number_t origin; + + /* slab_size == (1 << slab_size_shift) */ + unsigned int slab_size_shift; + + /* Determines how slabs should be queued during load */ + enum slab_depot_load_type load_type; + + /* The state for notifying slab journals to release recovery journal */ + sequence_number_t active_release_request; + sequence_number_t new_release_request; + + /* State variables for scrubbing complete handling */ + atomic_t zones_to_scrub; + + /* Array of pointers to individually allocated slabs */ + struct vdo_slab **slabs; + /* The number of slabs currently allocated and stored in 'slabs' */ + slab_count_t slab_count; + + /* Array of pointers to a larger set of slabs (used during resize) */ + struct vdo_slab **new_slabs; + /* The number of slabs currently allocated and stored in 'new_slabs' */ + slab_count_t new_slab_count; + /* The size that 'new_slabs' was allocated for */ + block_count_t new_size; + + /* The last block before resize, for rollback */ + physical_block_number_t old_last_block; + /* The last block after resize, for resize */ + physical_block_number_t new_last_block; + + /* The statistics for the slab summary */ + struct atomic_slab_summary_statistics summary_statistics; + /* The start of the slab summary partition */ + physical_block_number_t summary_origin; + /* The number of bits to shift to get a 7-bit fullness hint */ + unsigned int hint_shift; + /* The slab summary entries for all of the zones the partition can hold */ + struct slab_summary_entry *summary_entries; + + /* The block allocators for this depot */ + struct block_allocator allocators[]; +}; + +struct reference_updater; + +bool __must_check vdo_attempt_replay_into_slab(struct vdo_slab *slab, + physical_block_number_t pbn, + enum journal_operation operation, + bool increment, + struct journal_point *recovery_point, + struct vdo_completion *parent); + +int __must_check vdo_adjust_reference_count_for_rebuild(struct slab_depot *depot, + physical_block_number_t pbn, + enum journal_operation operation); + +static inline struct block_allocator *vdo_as_block_allocator(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_BLOCK_ALLOCATOR_COMPLETION); + return container_of(completion, struct block_allocator, completion); +} + +int __must_check vdo_acquire_provisional_reference(struct vdo_slab *slab, + physical_block_number_t pbn, + struct pbn_lock *lock); + +int __must_check vdo_allocate_block(struct block_allocator *allocator, + physical_block_number_t *block_number_ptr); + +int vdo_enqueue_clean_slab_waiter(struct block_allocator *allocator, + struct vdo_waiter *waiter); + +void vdo_modify_reference_count(struct vdo_completion *completion, + struct reference_updater *updater); + +int __must_check vdo_release_block_reference(struct block_allocator *allocator, + physical_block_number_t pbn); + +void vdo_notify_slab_journals_are_recovered(struct vdo_completion *completion); + +void vdo_dump_block_allocator(const struct block_allocator *allocator); + +int __must_check vdo_decode_slab_depot(struct slab_depot_state_2_0 state, + struct vdo *vdo, + struct partition *summary_partition, + struct slab_depot **depot_ptr); + +void vdo_free_slab_depot(struct slab_depot *depot); + +struct slab_depot_state_2_0 __must_check vdo_record_slab_depot(const struct slab_depot *depot); + +int __must_check vdo_allocate_reference_counters(struct slab_depot *depot); + +struct vdo_slab * __must_check vdo_get_slab(const struct slab_depot *depot, + physical_block_number_t pbn); + +u8 __must_check vdo_get_increment_limit(struct slab_depot *depot, + physical_block_number_t pbn); + +bool __must_check vdo_is_physical_data_block(const struct slab_depot *depot, + physical_block_number_t pbn); + +block_count_t __must_check vdo_get_slab_depot_allocated_blocks(const struct slab_depot *depot); + +block_count_t __must_check vdo_get_slab_depot_data_blocks(const struct slab_depot *depot); + +void vdo_get_slab_depot_statistics(const struct slab_depot *depot, + struct vdo_statistics *stats); + +void vdo_load_slab_depot(struct slab_depot *depot, + const struct admin_state_code *operation, + struct vdo_completion *parent, void *context); + +void vdo_prepare_slab_depot_to_allocate(struct slab_depot *depot, + enum slab_depot_load_type load_type, + struct vdo_completion *parent); + +void vdo_update_slab_depot_size(struct slab_depot *depot); + +int __must_check vdo_prepare_to_grow_slab_depot(struct slab_depot *depot, + const struct partition *partition); + +void vdo_use_new_slabs(struct slab_depot *depot, struct vdo_completion *parent); + +void vdo_abandon_new_slabs(struct slab_depot *depot); + +void vdo_drain_slab_depot(struct slab_depot *depot, + const struct admin_state_code *operation, + struct vdo_completion *parent); + +void vdo_resume_slab_depot(struct slab_depot *depot, struct vdo_completion *parent); + +void vdo_commit_oldest_slab_journal_tail_blocks(struct slab_depot *depot, + sequence_number_t recovery_block_number); + +void vdo_scrub_all_unrecovered_slabs(struct slab_depot *depot, + struct vdo_completion *parent); + +void vdo_dump_slab_depot(const struct slab_depot *depot); + +#endif /* VDO_SLAB_DEPOT_H */ diff --git a/drivers/md/dm-vdo/statistics.h b/drivers/md/dm-vdo/statistics.h new file mode 100644 index 000000000000..c88a75dffba3 --- /dev/null +++ b/drivers/md/dm-vdo/statistics.h @@ -0,0 +1,278 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef STATISTICS_H +#define STATISTICS_H + +#include "types.h" + +enum { + STATISTICS_VERSION = 36, +}; + +struct block_allocator_statistics { + /* The total number of slabs from which blocks may be allocated */ + u64 slab_count; + /* The total number of slabs from which blocks have ever been allocated */ + u64 slabs_opened; + /* The number of times since loading that a slab has been re-opened */ + u64 slabs_reopened; +}; + +/** + * Counters for tracking the number of items written (blocks, requests, etc.) + * that keep track of totals at steps in the write pipeline. Three counters + * allow the number of buffered, in-memory items and the number of in-flight, + * unacknowledged writes to be derived, while still tracking totals for + * reporting purposes + */ +struct commit_statistics { + /* The total number of items on which processing has started */ + u64 started; + /* The total number of items for which a write operation has been issued */ + u64 written; + /* The total number of items for which a write operation has completed */ + u64 committed; +}; + +/** Counters for events in the recovery journal */ +struct recovery_journal_statistics { + /* Number of times the on-disk journal was full */ + u64 disk_full; + /* Number of times the recovery journal requested slab journal commits. */ + u64 slab_journal_commits_requested; + /* Write/Commit totals for individual journal entries */ + struct commit_statistics entries; + /* Write/Commit totals for journal blocks */ + struct commit_statistics blocks; +}; + +/** The statistics for the compressed block packer. */ +struct packer_statistics { + /* Number of compressed data items written since startup */ + u64 compressed_fragments_written; + /* Number of blocks containing compressed items written since startup */ + u64 compressed_blocks_written; + /* Number of VIOs that are pending in the packer */ + u64 compressed_fragments_in_packer; +}; + +/** The statistics for the slab journals. */ +struct slab_journal_statistics { + /* Number of times the on-disk journal was full */ + u64 disk_full_count; + /* Number of times an entry was added over the flush threshold */ + u64 flush_count; + /* Number of times an entry was added over the block threshold */ + u64 blocked_count; + /* Number of times a tail block was written */ + u64 blocks_written; + /* Number of times we had to wait for the tail to write */ + u64 tail_busy_count; +}; + +/** The statistics for the slab summary. */ +struct slab_summary_statistics { + /* Number of blocks written */ + u64 blocks_written; +}; + +/** The statistics for the reference counts. */ +struct ref_counts_statistics { + /* Number of reference blocks written */ + u64 blocks_written; +}; + +/** The statistics for the block map. */ +struct block_map_statistics { + /* number of dirty (resident) pages */ + u32 dirty_pages; + /* number of clean (resident) pages */ + u32 clean_pages; + /* number of free pages */ + u32 free_pages; + /* number of pages in failed state */ + u32 failed_pages; + /* number of pages incoming */ + u32 incoming_pages; + /* number of pages outgoing */ + u32 outgoing_pages; + /* how many times free page not avail */ + u32 cache_pressure; + /* number of get_vdo_page() calls for read */ + u64 read_count; + /* number of get_vdo_page() calls for write */ + u64 write_count; + /* number of times pages failed to read */ + u64 failed_reads; + /* number of times pages failed to write */ + u64 failed_writes; + /* number of gets that are reclaimed */ + u64 reclaimed; + /* number of gets for outgoing pages */ + u64 read_outgoing; + /* number of gets that were already there */ + u64 found_in_cache; + /* number of gets requiring discard */ + u64 discard_required; + /* number of gets enqueued for their page */ + u64 wait_for_page; + /* number of gets that have to fetch */ + u64 fetch_required; + /* number of page fetches */ + u64 pages_loaded; + /* number of page saves */ + u64 pages_saved; + /* the number of flushes issued */ + u64 flush_count; +}; + +/** The dedupe statistics from hash locks */ +struct hash_lock_statistics { + /* Number of times the UDS advice proved correct */ + u64 dedupe_advice_valid; + /* Number of times the UDS advice proved incorrect */ + u64 dedupe_advice_stale; + /* Number of writes with the same data as another in-flight write */ + u64 concurrent_data_matches; + /* Number of writes whose hash collided with an in-flight write */ + u64 concurrent_hash_collisions; + /* Current number of dedupe queries that are in flight */ + u32 curr_dedupe_queries; +}; + +/** Counts of error conditions in VDO. */ +struct error_statistics { + /* number of times VDO got an invalid dedupe advice PBN from UDS */ + u64 invalid_advice_pbn_count; + /* number of times a VIO completed with a VDO_NO_SPACE error */ + u64 no_space_error_count; + /* number of times a VIO completed with a VDO_READ_ONLY error */ + u64 read_only_error_count; +}; + +struct bio_stats { + /* Number of REQ_OP_READ bios */ + u64 read; + /* Number of REQ_OP_WRITE bios with data */ + u64 write; + /* Number of bios tagged with REQ_PREFLUSH and containing no data */ + u64 empty_flush; + /* Number of REQ_OP_DISCARD bios */ + u64 discard; + /* Number of bios tagged with REQ_PREFLUSH */ + u64 flush; + /* Number of bios tagged with REQ_FUA */ + u64 fua; +}; + +struct memory_usage { + /* Tracked bytes currently allocated. */ + u64 bytes_used; + /* Maximum tracked bytes allocated. */ + u64 peak_bytes_used; +}; + +/** UDS index statistics */ +struct index_statistics { + /* Number of records stored in the index */ + u64 entries_indexed; + /* Number of post calls that found an existing entry */ + u64 posts_found; + /* Number of post calls that added a new entry */ + u64 posts_not_found; + /* Number of query calls that found an existing entry */ + u64 queries_found; + /* Number of query calls that added a new entry */ + u64 queries_not_found; + /* Number of update calls that found an existing entry */ + u64 updates_found; + /* Number of update calls that added a new entry */ + u64 updates_not_found; + /* Number of entries discarded */ + u64 entries_discarded; +}; + +/** The statistics of the vdo service. */ +struct vdo_statistics { + u32 version; + /* Number of blocks used for data */ + u64 data_blocks_used; + /* Number of blocks used for VDO metadata */ + u64 overhead_blocks_used; + /* Number of logical blocks that are currently mapped to physical blocks */ + u64 logical_blocks_used; + /* number of physical blocks */ + block_count_t physical_blocks; + /* number of logical blocks */ + block_count_t logical_blocks; + /* Size of the block map page cache, in bytes */ + u64 block_map_cache_size; + /* The physical block size */ + u64 block_size; + /* Number of times the VDO has successfully recovered */ + u64 complete_recoveries; + /* Number of times the VDO has recovered from read-only mode */ + u64 read_only_recoveries; + /* String describing the operating mode of the VDO */ + char mode[15]; + /* Whether the VDO is in recovery mode */ + bool in_recovery_mode; + /* What percentage of recovery mode work has been completed */ + u8 recovery_percentage; + /* The statistics for the compressed block packer */ + struct packer_statistics packer; + /* Counters for events in the block allocator */ + struct block_allocator_statistics allocator; + /* Counters for events in the recovery journal */ + struct recovery_journal_statistics journal; + /* The statistics for the slab journals */ + struct slab_journal_statistics slab_journal; + /* The statistics for the slab summary */ + struct slab_summary_statistics slab_summary; + /* The statistics for the reference counts */ + struct ref_counts_statistics ref_counts; + /* The statistics for the block map */ + struct block_map_statistics block_map; + /* The dedupe statistics from hash locks */ + struct hash_lock_statistics hash_lock; + /* Counts of error conditions */ + struct error_statistics errors; + /* The VDO instance */ + u32 instance; + /* Current number of active VIOs */ + u32 current_vios_in_progress; + /* Maximum number of active VIOs */ + u32 max_vios; + /* Number of times the UDS index was too slow in responding */ + u64 dedupe_advice_timeouts; + /* Number of flush requests submitted to the storage device */ + u64 flush_out; + /* Logical block size */ + u64 logical_block_size; + /* Bios submitted into VDO from above */ + struct bio_stats bios_in; + struct bio_stats bios_in_partial; + /* Bios submitted onward for user data */ + struct bio_stats bios_out; + /* Bios submitted onward for metadata */ + struct bio_stats bios_meta; + struct bio_stats bios_journal; + struct bio_stats bios_page_cache; + struct bio_stats bios_out_completed; + struct bio_stats bios_meta_completed; + struct bio_stats bios_journal_completed; + struct bio_stats bios_page_cache_completed; + struct bio_stats bios_acknowledged; + struct bio_stats bios_acknowledged_partial; + /* Current number of bios in progress */ + struct bio_stats bios_in_progress; + /* Memory usage stats. */ + struct memory_usage memory_usage; + /* The statistics for the UDS index */ + struct index_statistics index; +}; + +#endif /* not STATISTICS_H */ diff --git a/drivers/md/dm-vdo/status-codes.c b/drivers/md/dm-vdo/status-codes.c new file mode 100644 index 000000000000..d3493450b169 --- /dev/null +++ b/drivers/md/dm-vdo/status-codes.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "status-codes.h" + +#include "errors.h" +#include "logger.h" +#include "permassert.h" +#include "thread-utils.h" + +const struct error_info vdo_status_list[] = { + { "VDO_NOT_IMPLEMENTED", "Not implemented" }, + { "VDO_OUT_OF_RANGE", "Out of range" }, + { "VDO_REF_COUNT_INVALID", "Reference count would become invalid" }, + { "VDO_NO_SPACE", "Out of space" }, + { "VDO_BAD_CONFIGURATION", "Bad configuration option" }, + { "VDO_COMPONENT_BUSY", "Prior operation still in progress" }, + { "VDO_BAD_PAGE", "Corrupt or incorrect page" }, + { "VDO_UNSUPPORTED_VERSION", "Unsupported component version" }, + { "VDO_INCORRECT_COMPONENT", "Component id mismatch in decoder" }, + { "VDO_PARAMETER_MISMATCH", "Parameters have conflicting values" }, + { "VDO_UNKNOWN_PARTITION", "No partition exists with a given id" }, + { "VDO_PARTITION_EXISTS", "A partition already exists with a given id" }, + { "VDO_INCREMENT_TOO_SMALL", "Physical block growth of too few blocks" }, + { "VDO_CHECKSUM_MISMATCH", "Incorrect checksum" }, + { "VDO_LOCK_ERROR", "A lock is held incorrectly" }, + { "VDO_READ_ONLY", "The device is in read-only mode" }, + { "VDO_SHUTTING_DOWN", "The device is shutting down" }, + { "VDO_CORRUPT_JOURNAL", "Recovery journal entries corrupted" }, + { "VDO_TOO_MANY_SLABS", "Exceeds maximum number of slabs supported" }, + { "VDO_INVALID_FRAGMENT", "Compressed block fragment is invalid" }, + { "VDO_RETRY_AFTER_REBUILD", "Retry operation after rebuilding finishes" }, + { "VDO_BAD_MAPPING", "Invalid page mapping" }, + { "VDO_BIO_CREATION_FAILED", "Bio creation failed" }, + { "VDO_BAD_MAGIC", "Bad magic number" }, + { "VDO_BAD_NONCE", "Bad nonce" }, + { "VDO_JOURNAL_OVERFLOW", "Journal sequence number overflow" }, + { "VDO_INVALID_ADMIN_STATE", "Invalid operation for current state" }, +}; + +/** + * vdo_register_status_codes() - Register the VDO status codes. + * Return: A success or error code. + */ +int vdo_register_status_codes(void) +{ + int result; + + BUILD_BUG_ON((VDO_STATUS_CODE_LAST - VDO_STATUS_CODE_BASE) != + ARRAY_SIZE(vdo_status_list)); + + result = uds_register_error_block("VDO Status", VDO_STATUS_CODE_BASE, + VDO_STATUS_CODE_BLOCK_END, vdo_status_list, + sizeof(vdo_status_list)); + return (result == UDS_SUCCESS) ? VDO_SUCCESS : result; +} + +/** + * vdo_status_to_errno() - Given an error code, return a value we can return to the OS. + * @error: The error code to convert. + * + * The input error code may be a system-generated value (such as -EIO), an errno macro used in our + * code (such as EIO), or a UDS or VDO status code; the result must be something the rest of the OS + * can consume (negative errno values such as -EIO, in the case of the kernel). + * + * Return: A system error code value. + */ +int vdo_status_to_errno(int error) +{ + char error_name[VDO_MAX_ERROR_NAME_SIZE]; + char error_message[VDO_MAX_ERROR_MESSAGE_SIZE]; + + /* 0 is success, negative a system error code */ + if (likely(error <= 0)) + return error; + if (error < 1024) + return -error; + + /* VDO or UDS error */ + switch (error) { + case VDO_NO_SPACE: + return -ENOSPC; + case VDO_READ_ONLY: + return -EIO; + default: + vdo_log_info("%s: mapping internal status code %d (%s: %s) to EIO", + __func__, error, + uds_string_error_name(error, error_name, sizeof(error_name)), + uds_string_error(error, error_message, sizeof(error_message))); + return -EIO; + } +} diff --git a/drivers/md/dm-vdo/status-codes.h b/drivers/md/dm-vdo/status-codes.h new file mode 100644 index 000000000000..72da04159f88 --- /dev/null +++ b/drivers/md/dm-vdo/status-codes.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_STATUS_CODES_H +#define VDO_STATUS_CODES_H + +#include "errors.h" + +enum { + UDS_ERRORS_BLOCK_SIZE = UDS_ERROR_CODE_BLOCK_END - UDS_ERROR_CODE_BASE, + VDO_ERRORS_BLOCK_START = UDS_ERROR_CODE_BLOCK_END, + VDO_ERRORS_BLOCK_END = VDO_ERRORS_BLOCK_START + UDS_ERRORS_BLOCK_SIZE, +}; + +/* VDO-specific status codes. */ +enum vdo_status_codes { + /* base of all VDO errors */ + VDO_STATUS_CODE_BASE = VDO_ERRORS_BLOCK_START, + /* we haven't written this yet */ + VDO_NOT_IMPLEMENTED = VDO_STATUS_CODE_BASE, + /* input out of range */ + VDO_OUT_OF_RANGE, + /* an invalid reference count would result */ + VDO_REF_COUNT_INVALID, + /* a free block could not be allocated */ + VDO_NO_SPACE, + /* improper or missing configuration option */ + VDO_BAD_CONFIGURATION, + /* prior operation still in progress */ + VDO_COMPONENT_BUSY, + /* page contents incorrect or corrupt data */ + VDO_BAD_PAGE, + /* unsupported version of some component */ + VDO_UNSUPPORTED_VERSION, + /* component id mismatch in decoder */ + VDO_INCORRECT_COMPONENT, + /* parameters have conflicting values */ + VDO_PARAMETER_MISMATCH, + /* no partition exists with a given id */ + VDO_UNKNOWN_PARTITION, + /* a partition already exists with a given id */ + VDO_PARTITION_EXISTS, + /* physical block growth of too few blocks */ + VDO_INCREMENT_TOO_SMALL, + /* incorrect checksum */ + VDO_CHECKSUM_MISMATCH, + /* a lock is held incorrectly */ + VDO_LOCK_ERROR, + /* the VDO is in read-only mode */ + VDO_READ_ONLY, + /* the VDO is shutting down */ + VDO_SHUTTING_DOWN, + /* the recovery journal has corrupt entries */ + VDO_CORRUPT_JOURNAL, + /* exceeds maximum number of slabs supported */ + VDO_TOO_MANY_SLABS, + /* a compressed block fragment is invalid */ + VDO_INVALID_FRAGMENT, + /* action is unsupported while rebuilding */ + VDO_RETRY_AFTER_REBUILD, + /* a block map entry is invalid */ + VDO_BAD_MAPPING, + /* bio_add_page failed */ + VDO_BIO_CREATION_FAILED, + /* bad magic number */ + VDO_BAD_MAGIC, + /* bad nonce */ + VDO_BAD_NONCE, + /* sequence number overflow */ + VDO_JOURNAL_OVERFLOW, + /* the VDO is not in a state to perform an admin operation */ + VDO_INVALID_ADMIN_STATE, + /* one more than last error code */ + VDO_STATUS_CODE_LAST, + VDO_STATUS_CODE_BLOCK_END = VDO_ERRORS_BLOCK_END +}; + +extern const struct error_info vdo_status_list[]; + +int vdo_register_status_codes(void); + +int vdo_status_to_errno(int error); + +#endif /* VDO_STATUS_CODES_H */ diff --git a/drivers/md/dm-vdo/string-utils.c b/drivers/md/dm-vdo/string-utils.c new file mode 100644 index 000000000000..71e44b4683ea --- /dev/null +++ b/drivers/md/dm-vdo/string-utils.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "string-utils.h" + +char *vdo_append_to_buffer(char *buffer, char *buf_end, const char *fmt, ...) +{ + va_list args; + size_t n; + + va_start(args, fmt); + n = vsnprintf(buffer, buf_end - buffer, fmt, args); + if (n >= (size_t) (buf_end - buffer)) + buffer = buf_end; + else + buffer += n; + va_end(args); + + return buffer; +} diff --git a/drivers/md/dm-vdo/string-utils.h b/drivers/md/dm-vdo/string-utils.h new file mode 100644 index 000000000000..96eecd38b1c2 --- /dev/null +++ b/drivers/md/dm-vdo/string-utils.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_STRING_UTILS_H +#define VDO_STRING_UTILS_H + +#include +#include + +/* Utilities related to string manipulation */ + +static inline const char *vdo_bool_to_string(bool value) +{ + return value ? "true" : "false"; +} + +/* Append a formatted string to the end of a buffer. */ +char *vdo_append_to_buffer(char *buffer, char *buf_end, const char *fmt, ...) + __printf(3, 4); + +#endif /* VDO_STRING_UTILS_H */ diff --git a/drivers/md/dm-vdo/thread-device.c b/drivers/md/dm-vdo/thread-device.c new file mode 100644 index 000000000000..df13ca914db8 --- /dev/null +++ b/drivers/md/dm-vdo/thread-device.c @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "thread-device.h" + +/* A registry of threads associated with device id numbers. */ +static struct thread_registry device_id_thread_registry; + +/* Any registered thread must be unregistered. */ +void vdo_register_thread_device_id(struct registered_thread *new_thread, + unsigned int *id_ptr) +{ + vdo_register_thread(&device_id_thread_registry, new_thread, id_ptr); +} + +void vdo_unregister_thread_device_id(void) +{ + vdo_unregister_thread(&device_id_thread_registry); +} + +int vdo_get_thread_device_id(void) +{ + const unsigned int *pointer; + + pointer = vdo_lookup_thread(&device_id_thread_registry); + return (pointer != NULL) ? *pointer : -1; +} + +void vdo_initialize_thread_device_registry(void) +{ + vdo_initialize_thread_registry(&device_id_thread_registry); +} diff --git a/drivers/md/dm-vdo/thread-device.h b/drivers/md/dm-vdo/thread-device.h new file mode 100644 index 000000000000..494d9c9ef3f6 --- /dev/null +++ b/drivers/md/dm-vdo/thread-device.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_THREAD_DEVICE_H +#define VDO_THREAD_DEVICE_H + +#include "thread-registry.h" + +void vdo_register_thread_device_id(struct registered_thread *new_thread, + unsigned int *id_ptr); + +void vdo_unregister_thread_device_id(void); + +int vdo_get_thread_device_id(void); + +void vdo_initialize_thread_device_registry(void); + +#endif /* VDO_THREAD_DEVICE_H */ diff --git a/drivers/md/dm-vdo/thread-registry.c b/drivers/md/dm-vdo/thread-registry.c new file mode 100644 index 000000000000..d4a077d58c60 --- /dev/null +++ b/drivers/md/dm-vdo/thread-registry.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "thread-registry.h" + +#include +#include + +#include "permassert.h" + +/* + * We need to be careful when using other facilities that may use thread registry functions in + * their normal operation. For example, we do not want to invoke the logger while holding a lock. + */ + +void vdo_initialize_thread_registry(struct thread_registry *registry) +{ + INIT_LIST_HEAD(®istry->links); + spin_lock_init(®istry->lock); +} + +/* Register the current thread and associate it with a data pointer. */ +void vdo_register_thread(struct thread_registry *registry, + struct registered_thread *new_thread, const void *pointer) +{ + struct registered_thread *thread; + bool found_it = false; + + INIT_LIST_HEAD(&new_thread->links); + new_thread->pointer = pointer; + new_thread->task = current; + + spin_lock(®istry->lock); + list_for_each_entry(thread, ®istry->links, links) { + if (thread->task == current) { + /* There should be no existing entry. */ + list_del_rcu(&thread->links); + found_it = true; + break; + } + } + list_add_tail_rcu(&new_thread->links, ®istry->links); + spin_unlock(®istry->lock); + + VDO_ASSERT_LOG_ONLY(!found_it, "new thread not already in registry"); + if (found_it) { + /* Ensure no RCU iterators see it before re-initializing. */ + synchronize_rcu(); + INIT_LIST_HEAD(&thread->links); + } +} + +void vdo_unregister_thread(struct thread_registry *registry) +{ + struct registered_thread *thread; + bool found_it = false; + + spin_lock(®istry->lock); + list_for_each_entry(thread, ®istry->links, links) { + if (thread->task == current) { + list_del_rcu(&thread->links); + found_it = true; + break; + } + } + spin_unlock(®istry->lock); + + VDO_ASSERT_LOG_ONLY(found_it, "thread found in registry"); + if (found_it) { + /* Ensure no RCU iterators see it before re-initializing. */ + synchronize_rcu(); + INIT_LIST_HEAD(&thread->links); + } +} + +const void *vdo_lookup_thread(struct thread_registry *registry) +{ + struct registered_thread *thread; + const void *result = NULL; + + rcu_read_lock(); + list_for_each_entry_rcu(thread, ®istry->links, links) { + if (thread->task == current) { + result = thread->pointer; + break; + } + } + rcu_read_unlock(); + + return result; +} diff --git a/drivers/md/dm-vdo/thread-registry.h b/drivers/md/dm-vdo/thread-registry.h new file mode 100644 index 000000000000..cc6d78312b9e --- /dev/null +++ b/drivers/md/dm-vdo/thread-registry.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_THREAD_REGISTRY_H +#define VDO_THREAD_REGISTRY_H + +#include +#include + +struct thread_registry { + struct list_head links; + spinlock_t lock; +}; + +struct registered_thread { + struct list_head links; + const void *pointer; + struct task_struct *task; +}; + +void vdo_initialize_thread_registry(struct thread_registry *registry); + +void vdo_register_thread(struct thread_registry *registry, + struct registered_thread *new_thread, const void *pointer); + +void vdo_unregister_thread(struct thread_registry *registry); + +const void *vdo_lookup_thread(struct thread_registry *registry); + +#endif /* VDO_THREAD_REGISTRY_H */ diff --git a/drivers/md/dm-vdo/thread-utils.c b/drivers/md/dm-vdo/thread-utils.c new file mode 100644 index 000000000000..ec08478dd013 --- /dev/null +++ b/drivers/md/dm-vdo/thread-utils.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "thread-utils.h" + +#include +#include +#include +#include +#include + +#include "errors.h" +#include "logger.h" +#include "memory-alloc.h" + +static struct hlist_head thread_list; +static struct mutex thread_mutex; + +struct thread { + void (*thread_function)(void *thread_data); + void *thread_data; + struct hlist_node thread_links; + struct task_struct *thread_task; + struct completion thread_done; +}; + +void vdo_initialize_threads_mutex(void) +{ + mutex_init(&thread_mutex); +} + +static int thread_starter(void *arg) +{ + struct registered_thread allocating_thread; + struct thread *thread = arg; + + thread->thread_task = current; + mutex_lock(&thread_mutex); + hlist_add_head(&thread->thread_links, &thread_list); + mutex_unlock(&thread_mutex); + vdo_register_allocating_thread(&allocating_thread, NULL); + thread->thread_function(thread->thread_data); + vdo_unregister_allocating_thread(); + complete(&thread->thread_done); + return 0; +} + +int vdo_create_thread(void (*thread_function)(void *), void *thread_data, + const char *name, struct thread **new_thread) +{ + char *name_colon = strchr(name, ':'); + char *my_name_colon = strchr(current->comm, ':'); + struct task_struct *task; + struct thread *thread; + int result; + + result = vdo_allocate(1, struct thread, __func__, &thread); + if (result != VDO_SUCCESS) { + vdo_log_warning("Error allocating memory for %s", name); + return result; + } + + thread->thread_function = thread_function; + thread->thread_data = thread_data; + init_completion(&thread->thread_done); + /* + * Start the thread, with an appropriate thread name. + * + * If the name supplied contains a colon character, use that name. This causes uds module + * threads to have names like "uds:callbackW" and the main test runner thread to be named + * "zub:runtest". + * + * Otherwise if the current thread has a name containing a colon character, prefix the name + * supplied with the name of the current thread up to (and including) the colon character. + * Thus when the "kvdo0:dedupeQ" thread opens an index session, all the threads associated + * with that index will have names like "kvdo0:foo". + * + * Otherwise just use the name supplied. This should be a rare occurrence. + */ + if ((name_colon == NULL) && (my_name_colon != NULL)) { + task = kthread_run(thread_starter, thread, "%.*s:%s", + (int) (my_name_colon - current->comm), current->comm, + name); + } else { + task = kthread_run(thread_starter, thread, "%s", name); + } + + if (IS_ERR(task)) { + vdo_free(thread); + return PTR_ERR(task); + } + + *new_thread = thread; + return VDO_SUCCESS; +} + +void vdo_join_threads(struct thread *thread) +{ + while (wait_for_completion_interruptible(&thread->thread_done)) + fsleep(1000); + + mutex_lock(&thread_mutex); + hlist_del(&thread->thread_links); + mutex_unlock(&thread_mutex); + vdo_free(thread); +} diff --git a/drivers/md/dm-vdo/thread-utils.h b/drivers/md/dm-vdo/thread-utils.h new file mode 100644 index 000000000000..687ab43e2cee --- /dev/null +++ b/drivers/md/dm-vdo/thread-utils.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef THREAD_UTILS_H +#define THREAD_UTILS_H + +#include + +/* Thread and synchronization utilities */ + +struct thread; + +void vdo_initialize_threads_mutex(void); +int __must_check vdo_create_thread(void (*thread_function)(void *), void *thread_data, + const char *name, struct thread **new_thread); +void vdo_join_threads(struct thread *thread); + +#endif /* UDS_THREADS_H */ diff --git a/drivers/md/dm-vdo/time-utils.h b/drivers/md/dm-vdo/time-utils.h new file mode 100644 index 000000000000..5f1e850fd826 --- /dev/null +++ b/drivers/md/dm-vdo/time-utils.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef UDS_TIME_UTILS_H +#define UDS_TIME_UTILS_H + +#include +#include +#include + +static inline s64 ktime_to_seconds(ktime_t reltime) +{ + return reltime / NSEC_PER_SEC; +} + +static inline ktime_t current_time_ns(clockid_t clock) +{ + return clock == CLOCK_MONOTONIC ? ktime_get_ns() : ktime_get_real_ns(); +} + +static inline ktime_t current_time_us(void) +{ + return current_time_ns(CLOCK_REALTIME) / NSEC_PER_USEC; +} + +#endif /* UDS_TIME_UTILS_H */ diff --git a/drivers/md/dm-vdo/types.h b/drivers/md/dm-vdo/types.h new file mode 100644 index 000000000000..dbe892b10f26 --- /dev/null +++ b/drivers/md/dm-vdo/types.h @@ -0,0 +1,393 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_TYPES_H +#define VDO_TYPES_H + +#include +#include +#include +#include +#include +#include + +#include "funnel-queue.h" + +/* A size type in blocks. */ +typedef u64 block_count_t; + +/* The size of a block. */ +typedef u16 block_size_t; + +/* A counter for data_vios */ +typedef u16 data_vio_count_t; + +/* A height within a tree. */ +typedef u8 height_t; + +/* The logical block number as used by the consumer. */ +typedef u64 logical_block_number_t; + +/* The type of the nonce used to identify instances of VDO. */ +typedef u64 nonce_t; + +/* A size in pages. */ +typedef u32 page_count_t; + +/* A page number. */ +typedef u32 page_number_t; + +/* + * The physical (well, less logical) block number at which the block is found on the underlying + * device. + */ +typedef u64 physical_block_number_t; + +/* A count of tree roots. */ +typedef u8 root_count_t; + +/* A number of sectors. */ +typedef u8 sector_count_t; + +/* A sequence number. */ +typedef u64 sequence_number_t; + +/* The offset of a block within a slab. */ +typedef u32 slab_block_number; + +/* A size type in slabs. */ +typedef u16 slab_count_t; + +/* A slot in a bin or block map page. */ +typedef u16 slot_number_t; + +/* typedef thread_count_t - A thread counter. */ +typedef u8 thread_count_t; + +/* typedef thread_id_t - A thread ID, vdo threads are numbered sequentially from 0. */ +typedef u8 thread_id_t; + +/* A zone counter */ +typedef u8 zone_count_t; + +/* The following enums are persisted on storage, so the values must be preserved. */ + +/* The current operating mode of the VDO. */ +enum vdo_state { + VDO_DIRTY = 0, + VDO_NEW = 1, + VDO_CLEAN = 2, + VDO_READ_ONLY_MODE = 3, + VDO_FORCE_REBUILD = 4, + VDO_RECOVERING = 5, + VDO_REPLAYING = 6, /* VDO_REPLAYING is never set anymore, but retained for upgrade */ + VDO_REBUILD_FOR_UPGRADE = 7, + + /* Keep VDO_STATE_COUNT at the bottom. */ + VDO_STATE_COUNT +}; + +/** + * vdo_state_requires_read_only_rebuild() - Check whether a vdo_state indicates + * that a read-only rebuild is required. + * @state: The vdo_state to check. + * + * Return: true if the state indicates a rebuild is required + */ +static inline bool __must_check vdo_state_requires_read_only_rebuild(enum vdo_state state) +{ + return ((state == VDO_FORCE_REBUILD) || (state == VDO_REBUILD_FOR_UPGRADE)); +} + +/** + * vdo_state_requires_recovery() - Check whether a vdo state indicates that recovery is needed. + * @state: The state to check. + * + * Return: true if the state indicates a recovery is required + */ +static inline bool __must_check vdo_state_requires_recovery(enum vdo_state state) +{ + return ((state == VDO_DIRTY) || (state == VDO_REPLAYING) || (state == VDO_RECOVERING)); +} + +/* + * The current operation on a physical block (from the point of view of the recovery journal, slab + * journals, and reference counts. + */ +enum journal_operation { + VDO_JOURNAL_DATA_REMAPPING = 0, + VDO_JOURNAL_BLOCK_MAP_REMAPPING = 1, +} __packed; + +/* Partition IDs encoded in the volume layout in the super block. */ +enum partition_id { + VDO_BLOCK_MAP_PARTITION = 0, + VDO_SLAB_DEPOT_PARTITION = 1, + VDO_RECOVERY_JOURNAL_PARTITION = 2, + VDO_SLAB_SUMMARY_PARTITION = 3, +} __packed; + +/* Metadata types for the vdo. */ +enum vdo_metadata_type { + VDO_METADATA_RECOVERY_JOURNAL = 1, + VDO_METADATA_SLAB_JOURNAL = 2, + VDO_METADATA_RECOVERY_JOURNAL_2 = 3, +} __packed; + +/* A position in the block map where a block map entry is stored. */ +struct block_map_slot { + physical_block_number_t pbn; + slot_number_t slot; +}; + +/* + * Four bits of each five-byte block map entry contain a mapping state value used to distinguish + * unmapped or discarded logical blocks (which are treated as mapped to the zero block) from entries + * that have been mapped to a physical block, including the zero block. + * + * FIXME: these should maybe be defines. + */ +enum block_mapping_state { + VDO_MAPPING_STATE_UNMAPPED = 0, /* Must be zero to be the default value */ + VDO_MAPPING_STATE_UNCOMPRESSED = 1, /* A normal (uncompressed) block */ + VDO_MAPPING_STATE_COMPRESSED_BASE = 2, /* Compressed in slot 0 */ + VDO_MAPPING_STATE_COMPRESSED_MAX = 15, /* Compressed in slot 13 */ +}; + +enum { + VDO_MAX_COMPRESSION_SLOTS = + (VDO_MAPPING_STATE_COMPRESSED_MAX - VDO_MAPPING_STATE_COMPRESSED_BASE + 1), +}; + + +struct data_location { + physical_block_number_t pbn; + enum block_mapping_state state; +}; + +/* The configuration of a single slab derived from the configured block size and slab size. */ +struct slab_config { + /* total number of blocks in the slab */ + block_count_t slab_blocks; + /* number of blocks available for data */ + block_count_t data_blocks; + /* number of blocks for reference counts */ + block_count_t reference_count_blocks; + /* number of blocks for the slab journal */ + block_count_t slab_journal_blocks; + /* + * Number of blocks after which the slab journal starts pushing out a reference_block for + * each new entry it receives. + */ + block_count_t slab_journal_flushing_threshold; + /* + * Number of blocks after which the slab journal pushes out all reference_blocks and makes + * all vios wait. + */ + block_count_t slab_journal_blocking_threshold; + /* Number of blocks after which the slab must be scrubbed before coming online. */ + block_count_t slab_journal_scrubbing_threshold; +} __packed; + +/* + * This structure is memcmp'd for equality. Keep it packed and don't add any fields that are not + * properly set in both extant and parsed configs. + */ +struct thread_count_config { + unsigned int bio_ack_threads; + unsigned int bio_threads; + unsigned int bio_rotation_interval; + unsigned int cpu_threads; + unsigned int logical_zones; + unsigned int physical_zones; + unsigned int hash_zones; +} __packed; + +struct device_config { + struct dm_target *owning_target; + struct dm_dev *owned_device; + struct vdo *vdo; + /* All configs referencing a layer are kept on a list in the layer */ + struct list_head config_list; + char *original_string; + unsigned int version; + char *parent_device_name; + block_count_t physical_blocks; + /* + * This is the number of logical blocks from VDO's internal point of view. It is the number + * of 4K blocks regardless of the value of the logical_block_size parameter below. + */ + block_count_t logical_blocks; + unsigned int logical_block_size; + unsigned int cache_size; + unsigned int block_map_maximum_age; + bool deduplication; + bool compression; + struct thread_count_config thread_counts; + block_count_t max_discard_blocks; +}; + +enum vdo_completion_type { + /* Keep VDO_UNSET_COMPLETION_TYPE at the top. */ + VDO_UNSET_COMPLETION_TYPE, + VDO_ACTION_COMPLETION, + VDO_ADMIN_COMPLETION, + VDO_BLOCK_ALLOCATOR_COMPLETION, + VDO_DATA_VIO_POOL_COMPLETION, + VDO_DECREMENT_COMPLETION, + VDO_FLUSH_COMPLETION, + VDO_FLUSH_NOTIFICATION_COMPLETION, + VDO_GENERATION_FLUSHED_COMPLETION, + VDO_HASH_ZONE_COMPLETION, + VDO_HASH_ZONES_COMPLETION, + VDO_LOCK_COUNTER_COMPLETION, + VDO_PAGE_COMPLETION, + VDO_READ_ONLY_MODE_COMPLETION, + VDO_REPAIR_COMPLETION, + VDO_SYNC_COMPLETION, + VIO_COMPLETION, +} __packed; + +struct vdo_completion; + +/** + * typedef vdo_action_fn - An asynchronous VDO operation. + * @completion: The completion of the operation. + */ +typedef void (*vdo_action_fn)(struct vdo_completion *completion); + +enum vdo_completion_priority { + BIO_ACK_Q_ACK_PRIORITY = 0, + BIO_ACK_Q_MAX_PRIORITY = 0, + BIO_Q_COMPRESSED_DATA_PRIORITY = 0, + BIO_Q_DATA_PRIORITY = 0, + BIO_Q_FLUSH_PRIORITY = 2, + BIO_Q_HIGH_PRIORITY = 2, + BIO_Q_METADATA_PRIORITY = 1, + BIO_Q_VERIFY_PRIORITY = 1, + BIO_Q_MAX_PRIORITY = 2, + CPU_Q_COMPLETE_VIO_PRIORITY = 0, + CPU_Q_COMPLETE_READ_PRIORITY = 0, + CPU_Q_COMPRESS_BLOCK_PRIORITY = 0, + CPU_Q_EVENT_REPORTER_PRIORITY = 0, + CPU_Q_HASH_BLOCK_PRIORITY = 0, + CPU_Q_MAX_PRIORITY = 0, + UDS_Q_PRIORITY = 0, + UDS_Q_MAX_PRIORITY = 0, + VDO_DEFAULT_Q_COMPLETION_PRIORITY = 1, + VDO_DEFAULT_Q_FLUSH_PRIORITY = 2, + VDO_DEFAULT_Q_MAP_BIO_PRIORITY = 0, + VDO_DEFAULT_Q_SYNC_PRIORITY = 2, + VDO_DEFAULT_Q_VIO_CALLBACK_PRIORITY = 1, + VDO_DEFAULT_Q_MAX_PRIORITY = 2, + /* The maximum allowable priority */ + VDO_WORK_Q_MAX_PRIORITY = 2, + /* A value which must be out of range for a valid priority */ + VDO_WORK_Q_DEFAULT_PRIORITY = VDO_WORK_Q_MAX_PRIORITY + 1, +}; + +struct vdo_completion { + /* The type of completion this is */ + enum vdo_completion_type type; + + /* + * true once the processing of the operation is complete. This flag should not + * be used by waiters external to the VDO base as it is used to gate calling the callback. + */ + bool complete; + + /* + * If true, queue this completion on the next callback invocation, even if it is already + * running on the correct thread. + */ + bool requeue; + + /* The ID of the thread which should run the next callback */ + thread_id_t callback_thread_id; + + /* The result of the operation */ + int result; + + /* The VDO on which this completion operates */ + struct vdo *vdo; + + /* The callback which will be called once the operation is complete */ + vdo_action_fn callback; + + /* Callback which, if set, will be called if an error result is set */ + vdo_action_fn error_handler; + + /* The parent object, if any, that spawned this completion */ + void *parent; + + /* Entry link for lock-free work queue */ + struct funnel_queue_entry work_queue_entry_link; + enum vdo_completion_priority priority; + struct vdo_work_queue *my_queue; +}; + +struct block_allocator; +struct data_vio; +struct vdo; +struct vdo_config; + +/* vio types for statistics and instrumentation. */ +enum vio_type { + VIO_TYPE_UNINITIALIZED = 0, + VIO_TYPE_DATA, + VIO_TYPE_BLOCK_ALLOCATOR, + VIO_TYPE_BLOCK_MAP, + VIO_TYPE_BLOCK_MAP_INTERIOR, + VIO_TYPE_GEOMETRY, + VIO_TYPE_PARTITION_COPY, + VIO_TYPE_RECOVERY_JOURNAL, + VIO_TYPE_SLAB_JOURNAL, + VIO_TYPE_SLAB_SUMMARY, + VIO_TYPE_SUPER_BLOCK, +} __packed; + +/* Priority levels for asynchronous I/O operations performed on a vio. */ +enum vio_priority { + VIO_PRIORITY_LOW = 0, + VIO_PRIORITY_DATA = VIO_PRIORITY_LOW, + VIO_PRIORITY_COMPRESSED_DATA = VIO_PRIORITY_DATA, + VIO_PRIORITY_METADATA, + VIO_PRIORITY_HIGH, +} __packed; + +/* + * A wrapper for a bio. All I/O to the storage below a vdo is conducted via vios. + */ +struct vio { + /* The completion for this vio */ + struct vdo_completion completion; + + /* The bio zone in which I/O should be processed */ + zone_count_t bio_zone; + + /* The queueing priority of the vio operation */ + enum vio_priority priority; + + /* The vio type is used for statistics and instrumentation. */ + enum vio_type type; + + /* The size of this vio in blocks */ + unsigned int block_count; + + /* The data being read or written. */ + char *data; + + /* The VDO-owned bio to use for all IO for this vio */ + struct bio *bio; + + /* + * A list of enqueued bios with consecutive block numbers, stored by vdo_submit_bio() under + * the first-enqueued vio. The other vios are found via their bio entries in this list, and + * are not added to the work queue as separate completions. + */ + struct bio_list bios_merged; +}; + +#endif /* VDO_TYPES_H */ diff --git a/drivers/md/dm-vdo/vdo.c b/drivers/md/dm-vdo/vdo.c new file mode 100644 index 000000000000..fff847767755 --- /dev/null +++ b/drivers/md/dm-vdo/vdo.c @@ -0,0 +1,1730 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +/* + * This file contains the main entry points for normal operations on a vdo as well as functions for + * constructing and destroying vdo instances (in memory). + */ + +/** + * DOC: + * + * A read_only_notifier has a single completion which is used to perform read-only notifications, + * however, vdo_enter_read_only_mode() may be called from any thread. A pair of fields, protected + * by a spinlock, are used to control the read-only mode entry process. The first field holds the + * read-only error. The second is the state field, which may hold any of the four special values + * enumerated here. + * + * When vdo_enter_read_only_mode() is called from some vdo thread, if the read_only_error field + * already contains an error (i.e. its value is not VDO_SUCCESS), then some other error has already + * initiated the read-only process, and nothing more is done. Otherwise, the new error is stored in + * the read_only_error field, and the state field is consulted. If the state is MAY_NOTIFY, it is + * set to NOTIFYING, and the notification process begins. If the state is MAY_NOT_NOTIFY, then + * notifications are currently disallowed, generally due to the vdo being suspended. In this case, + * the nothing more will be done until the vdo is resumed, at which point the notification will be + * performed. In any other case, the vdo is already read-only, and there is nothing more to do. + */ + +#include "vdo.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" +#include "string-utils.h" + +#include "block-map.h" +#include "completion.h" +#include "data-vio.h" +#include "dedupe.h" +#include "encodings.h" +#include "funnel-workqueue.h" +#include "io-submitter.h" +#include "logical-zone.h" +#include "packer.h" +#include "physical-zone.h" +#include "recovery-journal.h" +#include "slab-depot.h" +#include "statistics.h" +#include "status-codes.h" +#include "vio.h" + +#define PARANOID_THREAD_CONSISTENCY_CHECKS 0 + +struct sync_completion { + struct vdo_completion vdo_completion; + struct completion completion; +}; + +/* A linked list is adequate for the small number of entries we expect. */ +struct device_registry { + struct list_head links; + /* TODO: Convert to rcu per kernel recommendation. */ + rwlock_t lock; +}; + +static struct device_registry registry; + +/** + * vdo_initialize_device_registry_once() - Initialize the necessary structures for the device + * registry. + */ +void vdo_initialize_device_registry_once(void) +{ + INIT_LIST_HEAD(®istry.links); + rwlock_init(®istry.lock); +} + +/** vdo_is_equal() - Implements vdo_filter_fn. */ +static bool vdo_is_equal(struct vdo *vdo, const void *context) +{ + return (vdo == context); +} + +/** + * filter_vdos_locked() - Find a vdo in the registry if it exists there. + * @filter: The filter function to apply to devices. + * @context: A bit of context to provide the filter. + * + * Context: Must be called holding the lock. + * + * Return: the vdo object found, if any. + */ +static struct vdo * __must_check filter_vdos_locked(vdo_filter_fn filter, + const void *context) +{ + struct vdo *vdo; + + list_for_each_entry(vdo, ®istry.links, registration) { + if (filter(vdo, context)) + return vdo; + } + + return NULL; +} + +/** + * vdo_find_matching() - Find and return the first (if any) vdo matching a given filter function. + * @filter: The filter function to apply to vdos. + * @context: A bit of context to provide the filter. + */ +struct vdo *vdo_find_matching(vdo_filter_fn filter, const void *context) +{ + struct vdo *vdo; + + read_lock(®istry.lock); + vdo = filter_vdos_locked(filter, context); + read_unlock(®istry.lock); + + return vdo; +} + +static void start_vdo_request_queue(void *ptr) +{ + struct vdo_thread *thread = vdo_get_work_queue_owner(vdo_get_current_work_queue()); + + vdo_register_allocating_thread(&thread->allocating_thread, + &thread->vdo->allocations_allowed); +} + +static void finish_vdo_request_queue(void *ptr) +{ + vdo_unregister_allocating_thread(); +} + +#ifdef MODULE +#define MODULE_NAME THIS_MODULE->name +#else +#define MODULE_NAME "dm-vdo" +#endif /* MODULE */ + +static const struct vdo_work_queue_type default_queue_type = { + .start = start_vdo_request_queue, + .finish = finish_vdo_request_queue, + .max_priority = VDO_DEFAULT_Q_MAX_PRIORITY, + .default_priority = VDO_DEFAULT_Q_COMPLETION_PRIORITY, +}; + +static const struct vdo_work_queue_type bio_ack_q_type = { + .start = NULL, + .finish = NULL, + .max_priority = BIO_ACK_Q_MAX_PRIORITY, + .default_priority = BIO_ACK_Q_ACK_PRIORITY, +}; + +static const struct vdo_work_queue_type cpu_q_type = { + .start = NULL, + .finish = NULL, + .max_priority = CPU_Q_MAX_PRIORITY, + .default_priority = CPU_Q_MAX_PRIORITY, +}; + +static void uninitialize_thread_config(struct thread_config *config) +{ + vdo_free(vdo_forget(config->logical_threads)); + vdo_free(vdo_forget(config->physical_threads)); + vdo_free(vdo_forget(config->hash_zone_threads)); + vdo_free(vdo_forget(config->bio_threads)); + memset(config, 0, sizeof(struct thread_config)); +} + +static void assign_thread_ids(struct thread_config *config, + thread_id_t thread_ids[], zone_count_t count) +{ + zone_count_t zone; + + for (zone = 0; zone < count; zone++) + thread_ids[zone] = config->thread_count++; +} + +/** + * initialize_thread_config() - Initialize the thread mapping + * + * If the logical, physical, and hash zone counts are all 0, a single thread will be shared by all + * three plus the packer and recovery journal. Otherwise, there must be at least one of each type, + * and each will have its own thread, as will the packer and recovery journal. + * + * Return: VDO_SUCCESS or an error. + */ +static int __must_check initialize_thread_config(struct thread_count_config counts, + struct thread_config *config) +{ + int result; + bool single = ((counts.logical_zones + counts.physical_zones + counts.hash_zones) == 0); + + config->bio_thread_count = counts.bio_threads; + if (single) { + config->logical_zone_count = 1; + config->physical_zone_count = 1; + config->hash_zone_count = 1; + } else { + config->logical_zone_count = counts.logical_zones; + config->physical_zone_count = counts.physical_zones; + config->hash_zone_count = counts.hash_zones; + } + + result = vdo_allocate(config->logical_zone_count, thread_id_t, + "logical thread array", &config->logical_threads); + if (result != VDO_SUCCESS) { + uninitialize_thread_config(config); + return result; + } + + result = vdo_allocate(config->physical_zone_count, thread_id_t, + "physical thread array", &config->physical_threads); + if (result != VDO_SUCCESS) { + uninitialize_thread_config(config); + return result; + } + + result = vdo_allocate(config->hash_zone_count, thread_id_t, + "hash thread array", &config->hash_zone_threads); + if (result != VDO_SUCCESS) { + uninitialize_thread_config(config); + return result; + } + + result = vdo_allocate(config->bio_thread_count, thread_id_t, + "bio thread array", &config->bio_threads); + if (result != VDO_SUCCESS) { + uninitialize_thread_config(config); + return result; + } + + if (single) { + config->logical_threads[0] = config->thread_count; + config->physical_threads[0] = config->thread_count; + config->hash_zone_threads[0] = config->thread_count++; + } else { + config->admin_thread = config->thread_count; + config->journal_thread = config->thread_count++; + config->packer_thread = config->thread_count++; + assign_thread_ids(config, config->logical_threads, counts.logical_zones); + assign_thread_ids(config, config->physical_threads, counts.physical_zones); + assign_thread_ids(config, config->hash_zone_threads, counts.hash_zones); + } + + config->dedupe_thread = config->thread_count++; + config->bio_ack_thread = + ((counts.bio_ack_threads > 0) ? config->thread_count++ : VDO_INVALID_THREAD_ID); + config->cpu_thread = config->thread_count++; + assign_thread_ids(config, config->bio_threads, counts.bio_threads); + return VDO_SUCCESS; +} + +/** + * read_geometry_block() - Synchronously read the geometry block from a vdo's underlying block + * device. + * @vdo: The vdo whose geometry is to be read. + * + * Return: VDO_SUCCESS or an error code. + */ +static int __must_check read_geometry_block(struct vdo *vdo) +{ + struct vio *vio; + char *block; + int result; + + result = vdo_allocate(VDO_BLOCK_SIZE, u8, __func__, &block); + if (result != VDO_SUCCESS) + return result; + + result = create_metadata_vio(vdo, VIO_TYPE_GEOMETRY, VIO_PRIORITY_HIGH, NULL, + block, &vio); + if (result != VDO_SUCCESS) { + vdo_free(block); + return result; + } + + /* + * This is only safe because, having not already loaded the geometry, the vdo's geometry's + * bio_offset field is 0, so the fact that vio_reset_bio() will subtract that offset from + * the supplied pbn is not a problem. + */ + result = vio_reset_bio(vio, block, NULL, REQ_OP_READ, + VDO_GEOMETRY_BLOCK_LOCATION); + if (result != VDO_SUCCESS) { + free_vio(vdo_forget(vio)); + vdo_free(block); + return result; + } + + bio_set_dev(vio->bio, vdo_get_backing_device(vdo)); + submit_bio_wait(vio->bio); + result = blk_status_to_errno(vio->bio->bi_status); + free_vio(vdo_forget(vio)); + if (result != 0) { + vdo_log_error_strerror(result, "synchronous read failed"); + vdo_free(block); + return -EIO; + } + + result = vdo_parse_geometry_block((u8 *) block, &vdo->geometry); + vdo_free(block); + return result; +} + +static bool get_zone_thread_name(const thread_id_t thread_ids[], zone_count_t count, + thread_id_t id, const char *prefix, + char *buffer, size_t buffer_length) +{ + if (id >= thread_ids[0]) { + thread_id_t index = id - thread_ids[0]; + + if (index < count) { + snprintf(buffer, buffer_length, "%s%d", prefix, index); + return true; + } + } + + return false; +} + +/** + * get_thread_name() - Format the name of the worker thread desired to support a given work queue. + * @thread_config: The thread configuration. + * @thread_id: The thread id. + * @buffer: Where to put the formatted name. + * @buffer_length: Size of the output buffer. + * + * The physical layer may add a prefix identifying the product; the output from this function + * should just identify the thread. + */ +static void get_thread_name(const struct thread_config *thread_config, + thread_id_t thread_id, char *buffer, size_t buffer_length) +{ + if (thread_id == thread_config->journal_thread) { + if (thread_config->packer_thread == thread_id) { + /* + * This is the "single thread" config where one thread is used for the + * journal, packer, logical, physical, and hash zones. In that case, it is + * known as the "request queue." + */ + snprintf(buffer, buffer_length, "reqQ"); + return; + } + + snprintf(buffer, buffer_length, "journalQ"); + return; + } else if (thread_id == thread_config->admin_thread) { + /* Theoretically this could be different from the journal thread. */ + snprintf(buffer, buffer_length, "adminQ"); + return; + } else if (thread_id == thread_config->packer_thread) { + snprintf(buffer, buffer_length, "packerQ"); + return; + } else if (thread_id == thread_config->dedupe_thread) { + snprintf(buffer, buffer_length, "dedupeQ"); + return; + } else if (thread_id == thread_config->bio_ack_thread) { + snprintf(buffer, buffer_length, "ackQ"); + return; + } else if (thread_id == thread_config->cpu_thread) { + snprintf(buffer, buffer_length, "cpuQ"); + return; + } + + if (get_zone_thread_name(thread_config->logical_threads, + thread_config->logical_zone_count, + thread_id, "logQ", buffer, buffer_length)) + return; + + if (get_zone_thread_name(thread_config->physical_threads, + thread_config->physical_zone_count, + thread_id, "physQ", buffer, buffer_length)) + return; + + if (get_zone_thread_name(thread_config->hash_zone_threads, + thread_config->hash_zone_count, + thread_id, "hashQ", buffer, buffer_length)) + return; + + if (get_zone_thread_name(thread_config->bio_threads, + thread_config->bio_thread_count, + thread_id, "bioQ", buffer, buffer_length)) + return; + + /* Some sort of misconfiguration? */ + snprintf(buffer, buffer_length, "reqQ%d", thread_id); +} + +/** + * vdo_make_thread() - Construct a single vdo work_queue and its associated thread (or threads for + * round-robin queues). + * @vdo: The vdo which owns the thread. + * @thread_id: The id of the thread to create (as determined by the thread_config). + * @type: The description of the work queue for this thread. + * @queue_count: The number of actual threads/queues contained in the "thread". + * @contexts: An array of queue_count contexts, one for each individual queue; may be NULL. + * + * Each "thread" constructed by this method is represented by a unique thread id in the thread + * config, and completions can be enqueued to the queue and run on the threads comprising this + * entity. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_make_thread(struct vdo *vdo, thread_id_t thread_id, + const struct vdo_work_queue_type *type, + unsigned int queue_count, void *contexts[]) +{ + struct vdo_thread *thread = &vdo->threads[thread_id]; + char queue_name[MAX_VDO_WORK_QUEUE_NAME_LEN]; + + if (type == NULL) + type = &default_queue_type; + + if (thread->queue != NULL) { + return VDO_ASSERT(vdo_work_queue_type_is(thread->queue, type), + "already constructed vdo thread %u is of the correct type", + thread_id); + } + + thread->vdo = vdo; + thread->thread_id = thread_id; + get_thread_name(&vdo->thread_config, thread_id, queue_name, sizeof(queue_name)); + return vdo_make_work_queue(vdo->thread_name_prefix, queue_name, thread, + type, queue_count, contexts, &thread->queue); +} + +/** + * register_vdo() - Register a VDO; it must not already be registered. + * @vdo: The vdo to register. + * + * Return: VDO_SUCCESS or an error. + */ +static int register_vdo(struct vdo *vdo) +{ + int result; + + write_lock(®istry.lock); + result = VDO_ASSERT(filter_vdos_locked(vdo_is_equal, vdo) == NULL, + "VDO not already registered"); + if (result == VDO_SUCCESS) { + INIT_LIST_HEAD(&vdo->registration); + list_add_tail(&vdo->registration, ®istry.links); + } + write_unlock(®istry.lock); + + return result; +} + +/** + * initialize_vdo() - Do the portion of initializing a vdo which will clean up after itself on + * error. + * @vdo: The vdo being initialized + * @config: The configuration of the vdo + * @instance: The instance number of the vdo + * @reason: The buffer to hold the failure reason on error + */ +static int initialize_vdo(struct vdo *vdo, struct device_config *config, + unsigned int instance, char **reason) +{ + int result; + zone_count_t i; + + vdo->device_config = config; + vdo->starting_sector_offset = config->owning_target->begin; + vdo->instance = instance; + vdo->allocations_allowed = true; + vdo_set_admin_state_code(&vdo->admin.state, VDO_ADMIN_STATE_NEW); + INIT_LIST_HEAD(&vdo->device_config_list); + vdo_initialize_completion(&vdo->admin.completion, vdo, VDO_ADMIN_COMPLETION); + init_completion(&vdo->admin.callback_sync); + mutex_init(&vdo->stats_mutex); + result = read_geometry_block(vdo); + if (result != VDO_SUCCESS) { + *reason = "Could not load geometry block"; + return result; + } + + result = initialize_thread_config(config->thread_counts, &vdo->thread_config); + if (result != VDO_SUCCESS) { + *reason = "Cannot create thread configuration"; + return result; + } + + vdo_log_info("zones: %d logical, %d physical, %d hash; total threads: %d", + config->thread_counts.logical_zones, + config->thread_counts.physical_zones, + config->thread_counts.hash_zones, vdo->thread_config.thread_count); + + /* Compression context storage */ + result = vdo_allocate(config->thread_counts.cpu_threads, char *, "LZ4 context", + &vdo->compression_context); + if (result != VDO_SUCCESS) { + *reason = "cannot allocate LZ4 context"; + return result; + } + + for (i = 0; i < config->thread_counts.cpu_threads; i++) { + result = vdo_allocate(LZ4_MEM_COMPRESS, char, "LZ4 context", + &vdo->compression_context[i]); + if (result != VDO_SUCCESS) { + *reason = "cannot allocate LZ4 context"; + return result; + } + } + + result = register_vdo(vdo); + if (result != VDO_SUCCESS) { + *reason = "Cannot add VDO to device registry"; + return result; + } + + vdo_set_admin_state_code(&vdo->admin.state, VDO_ADMIN_STATE_INITIALIZED); + return result; +} + +/** + * vdo_make() - Allocate and initialize a vdo. + * @instance: Device instantiation counter. + * @config: The device configuration. + * @reason: The reason for any failure during this call. + * @vdo_ptr: A pointer to hold the created vdo. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_make(unsigned int instance, struct device_config *config, char **reason, + struct vdo **vdo_ptr) +{ + int result; + struct vdo *vdo; + + /* Initialize with a generic failure reason to prevent returning garbage. */ + *reason = "Unspecified error"; + + result = vdo_allocate(1, struct vdo, __func__, &vdo); + if (result != VDO_SUCCESS) { + *reason = "Cannot allocate VDO"; + return result; + } + + result = initialize_vdo(vdo, config, instance, reason); + if (result != VDO_SUCCESS) { + vdo_destroy(vdo); + return result; + } + + /* From here on, the caller will clean up if there is an error. */ + *vdo_ptr = vdo; + + snprintf(vdo->thread_name_prefix, sizeof(vdo->thread_name_prefix), + "%s%u", MODULE_NAME, instance); + BUG_ON(vdo->thread_name_prefix[0] == '\0'); + result = vdo_allocate(vdo->thread_config.thread_count, + struct vdo_thread, __func__, &vdo->threads); + if (result != VDO_SUCCESS) { + *reason = "Cannot allocate thread structures"; + return result; + } + + result = vdo_make_thread(vdo, vdo->thread_config.admin_thread, + &default_queue_type, 1, NULL); + if (result != VDO_SUCCESS) { + *reason = "Cannot make admin thread"; + return result; + } + + result = vdo_make_flusher(vdo); + if (result != VDO_SUCCESS) { + *reason = "Cannot make flusher zones"; + return result; + } + + result = vdo_make_packer(vdo, DEFAULT_PACKER_BINS, &vdo->packer); + if (result != VDO_SUCCESS) { + *reason = "Cannot make packer zones"; + return result; + } + + BUG_ON(vdo->device_config->logical_block_size <= 0); + BUG_ON(vdo->device_config->owned_device == NULL); + result = make_data_vio_pool(vdo, MAXIMUM_VDO_USER_VIOS, + MAXIMUM_VDO_USER_VIOS * 3 / 4, + &vdo->data_vio_pool); + if (result != VDO_SUCCESS) { + *reason = "Cannot allocate data_vio pool"; + return result; + } + + result = vdo_make_io_submitter(config->thread_counts.bio_threads, + config->thread_counts.bio_rotation_interval, + get_data_vio_pool_request_limit(vdo->data_vio_pool), + vdo, &vdo->io_submitter); + if (result != VDO_SUCCESS) { + *reason = "bio submission initialization failed"; + return result; + } + + if (vdo_uses_bio_ack_queue(vdo)) { + result = vdo_make_thread(vdo, vdo->thread_config.bio_ack_thread, + &bio_ack_q_type, + config->thread_counts.bio_ack_threads, NULL); + if (result != VDO_SUCCESS) { + *reason = "bio ack queue initialization failed"; + return result; + } + } + + result = vdo_make_thread(vdo, vdo->thread_config.cpu_thread, &cpu_q_type, + config->thread_counts.cpu_threads, + (void **) vdo->compression_context); + if (result != VDO_SUCCESS) { + *reason = "CPU queue initialization failed"; + return result; + } + + return VDO_SUCCESS; +} + +static void finish_vdo(struct vdo *vdo) +{ + int i; + + if (vdo->threads == NULL) + return; + + vdo_cleanup_io_submitter(vdo->io_submitter); + vdo_finish_dedupe_index(vdo->hash_zones); + + for (i = 0; i < vdo->thread_config.thread_count; i++) + vdo_finish_work_queue(vdo->threads[i].queue); +} + +/** + * free_listeners() - Free the list of read-only listeners associated with a thread. + * @thread_data: The thread holding the list to free. + */ +static void free_listeners(struct vdo_thread *thread) +{ + struct read_only_listener *listener, *next; + + for (listener = vdo_forget(thread->listeners); listener != NULL; listener = next) { + next = vdo_forget(listener->next); + vdo_free(listener); + } +} + +static void uninitialize_super_block(struct vdo_super_block *super_block) +{ + free_vio_components(&super_block->vio); + vdo_free(super_block->buffer); +} + +/** + * unregister_vdo() - Remove a vdo from the device registry. + * @vdo: The vdo to remove. + */ +static void unregister_vdo(struct vdo *vdo) +{ + write_lock(®istry.lock); + if (filter_vdos_locked(vdo_is_equal, vdo) == vdo) + list_del_init(&vdo->registration); + + write_unlock(®istry.lock); +} + +/** + * vdo_destroy() - Destroy a vdo instance. + * @vdo: The vdo to destroy (may be NULL). + */ +void vdo_destroy(struct vdo *vdo) +{ + unsigned int i; + + if (vdo == NULL) + return; + + /* A running VDO should never be destroyed without suspending first. */ + BUG_ON(vdo_get_admin_state(vdo)->normal); + + vdo->allocations_allowed = true; + + finish_vdo(vdo); + unregister_vdo(vdo); + free_data_vio_pool(vdo->data_vio_pool); + vdo_free_io_submitter(vdo_forget(vdo->io_submitter)); + vdo_free_flusher(vdo_forget(vdo->flusher)); + vdo_free_packer(vdo_forget(vdo->packer)); + vdo_free_recovery_journal(vdo_forget(vdo->recovery_journal)); + vdo_free_slab_depot(vdo_forget(vdo->depot)); + vdo_uninitialize_layout(&vdo->layout); + vdo_uninitialize_layout(&vdo->next_layout); + if (vdo->partition_copier) + dm_kcopyd_client_destroy(vdo_forget(vdo->partition_copier)); + uninitialize_super_block(&vdo->super_block); + vdo_free_block_map(vdo_forget(vdo->block_map)); + vdo_free_hash_zones(vdo_forget(vdo->hash_zones)); + vdo_free_physical_zones(vdo_forget(vdo->physical_zones)); + vdo_free_logical_zones(vdo_forget(vdo->logical_zones)); + + if (vdo->threads != NULL) { + for (i = 0; i < vdo->thread_config.thread_count; i++) { + free_listeners(&vdo->threads[i]); + vdo_free_work_queue(vdo_forget(vdo->threads[i].queue)); + } + vdo_free(vdo_forget(vdo->threads)); + } + + uninitialize_thread_config(&vdo->thread_config); + + if (vdo->compression_context != NULL) { + for (i = 0; i < vdo->device_config->thread_counts.cpu_threads; i++) + vdo_free(vdo_forget(vdo->compression_context[i])); + + vdo_free(vdo_forget(vdo->compression_context)); + } + vdo_free(vdo); +} + +static int initialize_super_block(struct vdo *vdo, struct vdo_super_block *super_block) +{ + int result; + + result = vdo_allocate(VDO_BLOCK_SIZE, char, "encoded super block", + (char **) &vdo->super_block.buffer); + if (result != VDO_SUCCESS) + return result; + + return allocate_vio_components(vdo, VIO_TYPE_SUPER_BLOCK, + VIO_PRIORITY_METADATA, NULL, 1, + (char *) super_block->buffer, + &vdo->super_block.vio); +} + +/** + * finish_reading_super_block() - Continue after loading the super block. + * @completion: The super block vio. + * + * This callback is registered in vdo_load_super_block(). + */ +static void finish_reading_super_block(struct vdo_completion *completion) +{ + struct vdo_super_block *super_block = + container_of(as_vio(completion), struct vdo_super_block, vio); + + vdo_continue_completion(vdo_forget(completion->parent), + vdo_decode_super_block(super_block->buffer)); +} + +/** + * handle_super_block_read_error() - Handle an error reading the super block. + * @completion: The super block vio. + * + * This error handler is registered in vdo_load_super_block(). + */ +static void handle_super_block_read_error(struct vdo_completion *completion) +{ + vio_record_metadata_io_error(as_vio(completion)); + finish_reading_super_block(completion); +} + +static void read_super_block_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct vdo_completion *parent = vio->completion.parent; + + continue_vio_after_io(vio, finish_reading_super_block, + parent->callback_thread_id); +} + +/** + * vdo_load_super_block() - Allocate a super block and read its contents from storage. + * @vdo: The vdo containing the super block on disk. + * @parent: The completion to notify after loading the super block. + */ +void vdo_load_super_block(struct vdo *vdo, struct vdo_completion *parent) +{ + int result; + + result = initialize_super_block(vdo, &vdo->super_block); + if (result != VDO_SUCCESS) { + vdo_continue_completion(parent, result); + return; + } + + vdo->super_block.vio.completion.parent = parent; + vdo_submit_metadata_vio(&vdo->super_block.vio, + vdo_get_data_region_start(vdo->geometry), + read_super_block_endio, + handle_super_block_read_error, + REQ_OP_READ); +} + +/** + * vdo_get_backing_device() - Get the block device object underlying a vdo. + * @vdo: The vdo. + * + * Return: The vdo's current block device. + */ +struct block_device *vdo_get_backing_device(const struct vdo *vdo) +{ + return vdo->device_config->owned_device->bdev; +} + +/** + * vdo_get_device_name() - Get the device name associated with the vdo target. + * @target: The target device interface. + * + * Return: The block device name. + */ +const char *vdo_get_device_name(const struct dm_target *target) +{ + return dm_device_name(dm_table_get_md(target->table)); +} + +/** + * vdo_synchronous_flush() - Issue a flush request and wait for it to complete. + * @vdo: The vdo. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_synchronous_flush(struct vdo *vdo) +{ + int result; + struct bio bio; + + bio_init(&bio, vdo_get_backing_device(vdo), NULL, 0, + REQ_OP_WRITE | REQ_PREFLUSH); + submit_bio_wait(&bio); + result = blk_status_to_errno(bio.bi_status); + + atomic64_inc(&vdo->stats.flush_out); + if (result != 0) { + vdo_log_error_strerror(result, "synchronous flush failed"); + result = -EIO; + } + + bio_uninit(&bio); + return result; +} + +/** + * vdo_get_state() - Get the current state of the vdo. + * @vdo: The vdo. + + * Context: This method may be called from any thread. + * + * Return: The current state of the vdo. + */ +enum vdo_state vdo_get_state(const struct vdo *vdo) +{ + enum vdo_state state = atomic_read(&vdo->state); + + /* pairs with barriers where state field is changed */ + smp_rmb(); + return state; +} + +/** + * vdo_set_state() - Set the current state of the vdo. + * @vdo: The vdo whose state is to be set. + * @state: The new state of the vdo. + * + * Context: This method may be called from any thread. + */ +void vdo_set_state(struct vdo *vdo, enum vdo_state state) +{ + /* pairs with barrier in vdo_get_state */ + smp_wmb(); + atomic_set(&vdo->state, state); +} + +/** + * vdo_get_admin_state() - Get the admin state of the vdo. + * @vdo: The vdo. + * + * Return: The code for the vdo's current admin state. + */ +const struct admin_state_code *vdo_get_admin_state(const struct vdo *vdo) +{ + return vdo_get_admin_state_code(&vdo->admin.state); +} + +/** + * record_vdo() - Record the state of the VDO for encoding in the super block. + */ +static void record_vdo(struct vdo *vdo) +{ + /* This is for backwards compatibility. */ + vdo->states.unused = vdo->geometry.unused; + vdo->states.vdo.state = vdo_get_state(vdo); + vdo->states.block_map = vdo_record_block_map(vdo->block_map); + vdo->states.recovery_journal = vdo_record_recovery_journal(vdo->recovery_journal); + vdo->states.slab_depot = vdo_record_slab_depot(vdo->depot); + vdo->states.layout = vdo->layout; +} + +/** + * continue_super_block_parent() - Continue the parent of a super block save operation. + * @completion: The super block vio. + * + * This callback is registered in vdo_save_components(). + */ +static void continue_super_block_parent(struct vdo_completion *completion) +{ + vdo_continue_completion(vdo_forget(completion->parent), completion->result); +} + +/** + * handle_save_error() - Log a super block save error. + * @completion: The super block vio. + * + * This error handler is registered in vdo_save_components(). + */ +static void handle_save_error(struct vdo_completion *completion) +{ + struct vdo_super_block *super_block = + container_of(as_vio(completion), struct vdo_super_block, vio); + + vio_record_metadata_io_error(&super_block->vio); + vdo_log_error_strerror(completion->result, "super block save failed"); + /* + * Mark the super block as unwritable so that we won't attempt to write it again. This + * avoids the case where a growth attempt fails writing the super block with the new size, + * but the subsequent attempt to write out the read-only state succeeds. In this case, + * writes which happened just before the suspend would not be visible if the VDO is + * restarted without rebuilding, but, after a read-only rebuild, the effects of those + * writes would reappear. + */ + super_block->unwritable = true; + completion->callback(completion); +} + +static void super_block_write_endio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct vdo_completion *parent = vio->completion.parent; + + continue_vio_after_io(vio, continue_super_block_parent, + parent->callback_thread_id); +} + +/** + * vdo_save_components() - Encode the vdo and save the super block asynchronously. + * @vdo: The vdo whose state is being saved. + * @parent: The completion to notify when the save is complete. + */ +void vdo_save_components(struct vdo *vdo, struct vdo_completion *parent) +{ + struct vdo_super_block *super_block = &vdo->super_block; + + if (super_block->unwritable) { + vdo_continue_completion(parent, VDO_READ_ONLY); + return; + } + + if (super_block->vio.completion.parent != NULL) { + vdo_continue_completion(parent, VDO_COMPONENT_BUSY); + return; + } + + record_vdo(vdo); + + vdo_encode_super_block(super_block->buffer, &vdo->states); + super_block->vio.completion.parent = parent; + super_block->vio.completion.callback_thread_id = parent->callback_thread_id; + vdo_submit_metadata_vio(&super_block->vio, + vdo_get_data_region_start(vdo->geometry), + super_block_write_endio, handle_save_error, + REQ_OP_WRITE | REQ_PREFLUSH | REQ_FUA); +} + +/** + * vdo_register_read_only_listener() - Register a listener to be notified when the VDO goes + * read-only. + * @vdo: The vdo to register with. + * @listener: The object to notify. + * @notification: The function to call to send the notification. + * @thread_id: The id of the thread on which to send the notification. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_register_read_only_listener(struct vdo *vdo, void *listener, + vdo_read_only_notification_fn notification, + thread_id_t thread_id) +{ + struct vdo_thread *thread = &vdo->threads[thread_id]; + struct read_only_listener *read_only_listener; + int result; + + result = VDO_ASSERT(thread_id != vdo->thread_config.dedupe_thread, + "read only listener not registered on dedupe thread"); + if (result != VDO_SUCCESS) + return result; + + result = vdo_allocate(1, struct read_only_listener, __func__, + &read_only_listener); + if (result != VDO_SUCCESS) + return result; + + *read_only_listener = (struct read_only_listener) { + .listener = listener, + .notify = notification, + .next = thread->listeners, + }; + + thread->listeners = read_only_listener; + return VDO_SUCCESS; +} + +/** + * notify_vdo_of_read_only_mode() - Notify a vdo that it is going read-only. + * @listener: The vdo. + * @parent: The completion to notify in order to acknowledge the notification. + * + * This will save the read-only state to the super block. + * + * Implements vdo_read_only_notification_fn. + */ +static void notify_vdo_of_read_only_mode(void *listener, struct vdo_completion *parent) +{ + struct vdo *vdo = listener; + + if (vdo_in_read_only_mode(vdo)) + vdo_finish_completion(parent); + + vdo_set_state(vdo, VDO_READ_ONLY_MODE); + vdo_save_components(vdo, parent); +} + +/** + * vdo_enable_read_only_entry() - Enable a vdo to enter read-only mode on errors. + * @vdo: The vdo to enable. + * + * Return: VDO_SUCCESS or an error. + */ +int vdo_enable_read_only_entry(struct vdo *vdo) +{ + thread_id_t id; + bool is_read_only = vdo_in_read_only_mode(vdo); + struct read_only_notifier *notifier = &vdo->read_only_notifier; + + if (is_read_only) { + notifier->read_only_error = VDO_READ_ONLY; + notifier->state = NOTIFIED; + } else { + notifier->state = MAY_NOT_NOTIFY; + } + + spin_lock_init(¬ifier->lock); + vdo_initialize_completion(¬ifier->completion, vdo, + VDO_READ_ONLY_MODE_COMPLETION); + + for (id = 0; id < vdo->thread_config.thread_count; id++) + vdo->threads[id].is_read_only = is_read_only; + + return vdo_register_read_only_listener(vdo, vdo, notify_vdo_of_read_only_mode, + vdo->thread_config.admin_thread); +} + +/** + * vdo_wait_until_not_entering_read_only_mode() - Wait until no read-only notifications are in + * progress and prevent any subsequent + * notifications. + * @parent: The completion to notify when no threads are entering read-only mode. + * + * Notifications may be re-enabled by calling vdo_allow_read_only_mode_entry(). + */ +void vdo_wait_until_not_entering_read_only_mode(struct vdo_completion *parent) +{ + struct vdo *vdo = parent->vdo; + struct read_only_notifier *notifier = &vdo->read_only_notifier; + + vdo_assert_on_admin_thread(vdo, __func__); + + if (notifier->waiter != NULL) { + vdo_continue_completion(parent, VDO_COMPONENT_BUSY); + return; + } + + spin_lock(¬ifier->lock); + if (notifier->state == NOTIFYING) + notifier->waiter = parent; + else if (notifier->state == MAY_NOTIFY) + notifier->state = MAY_NOT_NOTIFY; + spin_unlock(¬ifier->lock); + + if (notifier->waiter == NULL) { + /* + * A notification was not in progress, and now they are + * disallowed. + */ + vdo_launch_completion(parent); + return; + } +} + +/** + * as_notifier() - Convert a generic vdo_completion to a read_only_notifier. + * @completion: The completion to convert. + * + * Return: The completion as a read_only_notifier. + */ +static inline struct read_only_notifier *as_notifier(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_READ_ONLY_MODE_COMPLETION); + return container_of(completion, struct read_only_notifier, completion); +} + +/** + * finish_entering_read_only_mode() - Complete the process of entering read only mode. + * @completion: The read-only mode completion. + */ +static void finish_entering_read_only_mode(struct vdo_completion *completion) +{ + struct read_only_notifier *notifier = as_notifier(completion); + + vdo_assert_on_admin_thread(completion->vdo, __func__); + + spin_lock(¬ifier->lock); + notifier->state = NOTIFIED; + spin_unlock(¬ifier->lock); + + if (notifier->waiter != NULL) + vdo_continue_completion(vdo_forget(notifier->waiter), + completion->result); +} + +/** + * make_thread_read_only() - Inform each thread that the VDO is in read-only mode. + * @completion: The read-only mode completion. + */ +static void make_thread_read_only(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + thread_id_t thread_id = completion->callback_thread_id; + struct read_only_notifier *notifier = as_notifier(completion); + struct read_only_listener *listener = completion->parent; + + if (listener == NULL) { + /* This is the first call on this thread */ + struct vdo_thread *thread = &vdo->threads[thread_id]; + + thread->is_read_only = true; + listener = thread->listeners; + if (thread_id == 0) + vdo_log_error_strerror(READ_ONCE(notifier->read_only_error), + "Unrecoverable error, entering read-only mode"); + } else { + /* We've just finished notifying a listener */ + listener = listener->next; + } + + if (listener != NULL) { + /* We have a listener to notify */ + vdo_prepare_completion(completion, make_thread_read_only, + make_thread_read_only, thread_id, + listener); + listener->notify(listener->listener, completion); + return; + } + + /* We're done with this thread */ + if (++thread_id == vdo->thread_config.dedupe_thread) { + /* + * We don't want to notify the dedupe thread since it may be + * blocked rebuilding the index. + */ + thread_id++; + } + + if (thread_id >= vdo->thread_config.thread_count) { + /* There are no more threads */ + vdo_prepare_completion(completion, finish_entering_read_only_mode, + finish_entering_read_only_mode, + vdo->thread_config.admin_thread, NULL); + } else { + vdo_prepare_completion(completion, make_thread_read_only, + make_thread_read_only, thread_id, NULL); + } + + vdo_launch_completion(completion); +} + +/** + * vdo_allow_read_only_mode_entry() - Allow the notifier to put the VDO into read-only mode, + * reversing the effects of + * vdo_wait_until_not_entering_read_only_mode(). + * @parent: The object to notify once the operation is complete. + * + * If some thread tried to put the vdo into read-only mode while notifications were disallowed, it + * will be done when this method is called. If that happens, the parent will not be notified until + * the vdo has actually entered read-only mode and attempted to save the super block. + * + * Context: This method may only be called from the admin thread. + */ +void vdo_allow_read_only_mode_entry(struct vdo_completion *parent) +{ + struct vdo *vdo = parent->vdo; + struct read_only_notifier *notifier = &vdo->read_only_notifier; + + vdo_assert_on_admin_thread(vdo, __func__); + + if (notifier->waiter != NULL) { + vdo_continue_completion(parent, VDO_COMPONENT_BUSY); + return; + } + + spin_lock(¬ifier->lock); + if (notifier->state == MAY_NOT_NOTIFY) { + if (notifier->read_only_error == VDO_SUCCESS) { + notifier->state = MAY_NOTIFY; + } else { + notifier->state = NOTIFYING; + notifier->waiter = parent; + } + } + spin_unlock(¬ifier->lock); + + if (notifier->waiter == NULL) { + /* We're done */ + vdo_launch_completion(parent); + return; + } + + /* Do the pending notification. */ + make_thread_read_only(¬ifier->completion); +} + +/** + * vdo_enter_read_only_mode() - Put a VDO into read-only mode and save the read-only state in the + * super block. + * @vdo: The vdo. + * @error_code: The error which caused the VDO to enter read-only mode. + * + * This method is a no-op if the VDO is already read-only. + */ +void vdo_enter_read_only_mode(struct vdo *vdo, int error_code) +{ + bool notify = false; + thread_id_t thread_id = vdo_get_callback_thread_id(); + struct read_only_notifier *notifier = &vdo->read_only_notifier; + struct vdo_thread *thread; + + if (thread_id != VDO_INVALID_THREAD_ID) { + thread = &vdo->threads[thread_id]; + if (thread->is_read_only) { + /* This thread has already gone read-only. */ + return; + } + + /* Record for this thread that the VDO is read-only. */ + thread->is_read_only = true; + } + + spin_lock(¬ifier->lock); + if (notifier->read_only_error == VDO_SUCCESS) { + WRITE_ONCE(notifier->read_only_error, error_code); + if (notifier->state == MAY_NOTIFY) { + notifier->state = NOTIFYING; + notify = true; + } + } + spin_unlock(¬ifier->lock); + + if (!notify) { + /* The notifier is already aware of a read-only error */ + return; + } + + /* Initiate a notification starting on the lowest numbered thread. */ + vdo_launch_completion_callback(¬ifier->completion, make_thread_read_only, 0); +} + +/** + * vdo_is_read_only() - Check whether the VDO is read-only. + * @vdo: The vdo. + * + * Return: true if the vdo is read-only. + * + * This method may be called from any thread, as opposed to examining the VDO's state field which + * is only safe to check from the admin thread. + */ +bool vdo_is_read_only(struct vdo *vdo) +{ + return vdo->threads[vdo_get_callback_thread_id()].is_read_only; +} + +/** + * vdo_in_read_only_mode() - Check whether a vdo is in read-only mode. + * @vdo: The vdo to query. + * + * Return: true if the vdo is in read-only mode. + */ +bool vdo_in_read_only_mode(const struct vdo *vdo) +{ + return (vdo_get_state(vdo) == VDO_READ_ONLY_MODE); +} + +/** + * vdo_in_recovery_mode() - Check whether the vdo is in recovery mode. + * @vdo: The vdo to query. + * + * Return: true if the vdo is in recovery mode. + */ +bool vdo_in_recovery_mode(const struct vdo *vdo) +{ + return (vdo_get_state(vdo) == VDO_RECOVERING); +} + +/** + * vdo_enter_recovery_mode() - Put the vdo into recovery mode. + * @vdo: The vdo. + */ +void vdo_enter_recovery_mode(struct vdo *vdo) +{ + vdo_assert_on_admin_thread(vdo, __func__); + + if (vdo_in_read_only_mode(vdo)) + return; + + vdo_log_info("Entering recovery mode"); + vdo_set_state(vdo, VDO_RECOVERING); +} + +/** + * complete_synchronous_action() - Signal the waiting thread that a synchronous action is complete. + * @completion: The sync completion. + */ +static void complete_synchronous_action(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VDO_SYNC_COMPLETION); + complete(&(container_of(completion, struct sync_completion, + vdo_completion)->completion)); +} + +/** + * perform_synchronous_action() - Launch an action on a VDO thread and wait for it to complete. + * @vdo: The vdo. + * @action: The callback to launch. + * @thread_id: The thread on which to run the action. + * @parent: The parent of the sync completion (may be NULL). + */ +static int perform_synchronous_action(struct vdo *vdo, vdo_action_fn action, + thread_id_t thread_id, void *parent) +{ + struct sync_completion sync; + + vdo_initialize_completion(&sync.vdo_completion, vdo, VDO_SYNC_COMPLETION); + init_completion(&sync.completion); + sync.vdo_completion.parent = parent; + vdo_launch_completion_callback(&sync.vdo_completion, action, thread_id); + wait_for_completion(&sync.completion); + return sync.vdo_completion.result; +} + +/** + * set_compression_callback() - Callback to turn compression on or off. + * @completion: The completion. + */ +static void set_compression_callback(struct vdo_completion *completion) +{ + struct vdo *vdo = completion->vdo; + bool *enable = completion->parent; + bool was_enabled = vdo_get_compressing(vdo); + + if (*enable != was_enabled) { + WRITE_ONCE(vdo->compressing, *enable); + if (was_enabled) { + /* Signal the packer to flush since compression has been disabled. */ + vdo_flush_packer(vdo->packer); + } + } + + vdo_log_info("compression is %s", (*enable ? "enabled" : "disabled")); + *enable = was_enabled; + complete_synchronous_action(completion); +} + +/** + * vdo_set_compressing() - Turn compression on or off. + * @vdo: The vdo. + * @enable: Whether to enable or disable compression. + * + * Return: Whether compression was previously on or off. + */ +bool vdo_set_compressing(struct vdo *vdo, bool enable) +{ + perform_synchronous_action(vdo, set_compression_callback, + vdo->thread_config.packer_thread, + &enable); + return enable; +} + +/** + * vdo_get_compressing() - Get whether compression is enabled in a vdo. + * @vdo: The vdo. + * + * Return: State of compression. + */ +bool vdo_get_compressing(struct vdo *vdo) +{ + return READ_ONCE(vdo->compressing); +} + +static size_t get_block_map_cache_size(const struct vdo *vdo) +{ + return ((size_t) vdo->device_config->cache_size) * VDO_BLOCK_SIZE; +} + +static struct error_statistics __must_check get_vdo_error_statistics(const struct vdo *vdo) +{ + /* + * The error counts can be incremented from arbitrary threads and so must be incremented + * atomically, but they are just statistics with no semantics that could rely on memory + * order, so unfenced reads are sufficient. + */ + const struct atomic_statistics *atoms = &vdo->stats; + + return (struct error_statistics) { + .invalid_advice_pbn_count = atomic64_read(&atoms->invalid_advice_pbn_count), + .no_space_error_count = atomic64_read(&atoms->no_space_error_count), + .read_only_error_count = atomic64_read(&atoms->read_only_error_count), + }; +} + +static void copy_bio_stat(struct bio_stats *b, const struct atomic_bio_stats *a) +{ + b->read = atomic64_read(&a->read); + b->write = atomic64_read(&a->write); + b->discard = atomic64_read(&a->discard); + b->flush = atomic64_read(&a->flush); + b->empty_flush = atomic64_read(&a->empty_flush); + b->fua = atomic64_read(&a->fua); +} + +static struct bio_stats subtract_bio_stats(struct bio_stats minuend, + struct bio_stats subtrahend) +{ + return (struct bio_stats) { + .read = minuend.read - subtrahend.read, + .write = minuend.write - subtrahend.write, + .discard = minuend.discard - subtrahend.discard, + .flush = minuend.flush - subtrahend.flush, + .empty_flush = minuend.empty_flush - subtrahend.empty_flush, + .fua = minuend.fua - subtrahend.fua, + }; +} + +/** + * vdo_get_physical_blocks_allocated() - Get the number of physical blocks in use by user data. + * @vdo: The vdo. + * + * Return: The number of blocks allocated for user data. + */ +static block_count_t __must_check vdo_get_physical_blocks_allocated(const struct vdo *vdo) +{ + return (vdo_get_slab_depot_allocated_blocks(vdo->depot) - + vdo_get_journal_block_map_data_blocks_used(vdo->recovery_journal)); +} + +/** + * vdo_get_physical_blocks_overhead() - Get the number of physical blocks used by vdo metadata. + * @vdo: The vdo. + * + * Return: The number of overhead blocks. + */ +static block_count_t __must_check vdo_get_physical_blocks_overhead(const struct vdo *vdo) +{ + /* + * config.physical_blocks is mutated during resize and is in a packed structure, + * but resize runs on admin thread. + * TODO: Verify that this is always safe. + */ + return (vdo->states.vdo.config.physical_blocks - + vdo_get_slab_depot_data_blocks(vdo->depot) + + vdo_get_journal_block_map_data_blocks_used(vdo->recovery_journal)); +} + +static const char *vdo_describe_state(enum vdo_state state) +{ + /* These strings should all fit in the 15 chars of VDOStatistics.mode. */ + switch (state) { + case VDO_RECOVERING: + return "recovering"; + + case VDO_READ_ONLY_MODE: + return "read-only"; + + default: + return "normal"; + } +} + +/** + * get_vdo_statistics() - Populate a vdo_statistics structure on the admin thread. + * @vdo: The vdo. + * @stats: The statistics structure to populate. + */ +static void get_vdo_statistics(const struct vdo *vdo, struct vdo_statistics *stats) +{ + struct recovery_journal *journal = vdo->recovery_journal; + enum vdo_state state = vdo_get_state(vdo); + + vdo_assert_on_admin_thread(vdo, __func__); + + /* start with a clean slate */ + memset(stats, 0, sizeof(struct vdo_statistics)); + + /* + * These are immutable properties of the vdo object, so it is safe to query them from any + * thread. + */ + stats->version = STATISTICS_VERSION; + stats->logical_blocks = vdo->states.vdo.config.logical_blocks; + /* + * config.physical_blocks is mutated during resize and is in a packed structure, but resize + * runs on the admin thread. + * TODO: verify that this is always safe + */ + stats->physical_blocks = vdo->states.vdo.config.physical_blocks; + stats->block_size = VDO_BLOCK_SIZE; + stats->complete_recoveries = vdo->states.vdo.complete_recoveries; + stats->read_only_recoveries = vdo->states.vdo.read_only_recoveries; + stats->block_map_cache_size = get_block_map_cache_size(vdo); + + /* The callees are responsible for thread-safety. */ + stats->data_blocks_used = vdo_get_physical_blocks_allocated(vdo); + stats->overhead_blocks_used = vdo_get_physical_blocks_overhead(vdo); + stats->logical_blocks_used = vdo_get_recovery_journal_logical_blocks_used(journal); + vdo_get_slab_depot_statistics(vdo->depot, stats); + stats->journal = vdo_get_recovery_journal_statistics(journal); + stats->packer = vdo_get_packer_statistics(vdo->packer); + stats->block_map = vdo_get_block_map_statistics(vdo->block_map); + vdo_get_dedupe_statistics(vdo->hash_zones, stats); + stats->errors = get_vdo_error_statistics(vdo); + stats->in_recovery_mode = (state == VDO_RECOVERING); + snprintf(stats->mode, sizeof(stats->mode), "%s", vdo_describe_state(state)); + + stats->instance = vdo->instance; + stats->current_vios_in_progress = get_data_vio_pool_active_requests(vdo->data_vio_pool); + stats->max_vios = get_data_vio_pool_maximum_requests(vdo->data_vio_pool); + + stats->flush_out = atomic64_read(&vdo->stats.flush_out); + stats->logical_block_size = vdo->device_config->logical_block_size; + copy_bio_stat(&stats->bios_in, &vdo->stats.bios_in); + copy_bio_stat(&stats->bios_in_partial, &vdo->stats.bios_in_partial); + copy_bio_stat(&stats->bios_out, &vdo->stats.bios_out); + copy_bio_stat(&stats->bios_meta, &vdo->stats.bios_meta); + copy_bio_stat(&stats->bios_journal, &vdo->stats.bios_journal); + copy_bio_stat(&stats->bios_page_cache, &vdo->stats.bios_page_cache); + copy_bio_stat(&stats->bios_out_completed, &vdo->stats.bios_out_completed); + copy_bio_stat(&stats->bios_meta_completed, &vdo->stats.bios_meta_completed); + copy_bio_stat(&stats->bios_journal_completed, + &vdo->stats.bios_journal_completed); + copy_bio_stat(&stats->bios_page_cache_completed, + &vdo->stats.bios_page_cache_completed); + copy_bio_stat(&stats->bios_acknowledged, &vdo->stats.bios_acknowledged); + copy_bio_stat(&stats->bios_acknowledged_partial, &vdo->stats.bios_acknowledged_partial); + stats->bios_in_progress = + subtract_bio_stats(stats->bios_in, stats->bios_acknowledged); + vdo_get_memory_stats(&stats->memory_usage.bytes_used, + &stats->memory_usage.peak_bytes_used); +} + +/** + * vdo_fetch_statistics_callback() - Action to populate a vdo_statistics + * structure on the admin thread. + * @completion: The completion. + * + * This callback is registered in vdo_fetch_statistics(). + */ +static void vdo_fetch_statistics_callback(struct vdo_completion *completion) +{ + get_vdo_statistics(completion->vdo, completion->parent); + complete_synchronous_action(completion); +} + +/** + * vdo_fetch_statistics() - Fetch statistics on the correct thread. + * @vdo: The vdo. + * @stats: The vdo statistics are returned here. + */ +void vdo_fetch_statistics(struct vdo *vdo, struct vdo_statistics *stats) +{ + perform_synchronous_action(vdo, vdo_fetch_statistics_callback, + vdo->thread_config.admin_thread, stats); +} + +/** + * vdo_get_callback_thread_id() - Get the id of the callback thread on which a completion is + * currently running. + * + * Return: The current thread ID, or -1 if no such thread. + */ +thread_id_t vdo_get_callback_thread_id(void) +{ + struct vdo_work_queue *queue = vdo_get_current_work_queue(); + struct vdo_thread *thread; + thread_id_t thread_id; + + if (queue == NULL) + return VDO_INVALID_THREAD_ID; + + thread = vdo_get_work_queue_owner(queue); + thread_id = thread->thread_id; + + if (PARANOID_THREAD_CONSISTENCY_CHECKS) { + BUG_ON(thread_id >= thread->vdo->thread_config.thread_count); + BUG_ON(thread != &thread->vdo->threads[thread_id]); + } + + return thread_id; +} + +/** + * vdo_dump_status() - Dump status information about a vdo to the log for debugging. + * @vdo: The vdo to dump. + */ +void vdo_dump_status(const struct vdo *vdo) +{ + zone_count_t zone; + + vdo_dump_flusher(vdo->flusher); + vdo_dump_recovery_journal_statistics(vdo->recovery_journal); + vdo_dump_packer(vdo->packer); + vdo_dump_slab_depot(vdo->depot); + + for (zone = 0; zone < vdo->thread_config.logical_zone_count; zone++) + vdo_dump_logical_zone(&vdo->logical_zones->zones[zone]); + + for (zone = 0; zone < vdo->thread_config.physical_zone_count; zone++) + vdo_dump_physical_zone(&vdo->physical_zones->zones[zone]); + + vdo_dump_hash_zones(vdo->hash_zones); +} + +/** + * vdo_assert_on_admin_thread() - Assert that we are running on the admin thread. + * @vdo: The vdo. + * @name: The name of the function which should be running on the admin thread (for logging). + */ +void vdo_assert_on_admin_thread(const struct vdo *vdo, const char *name) +{ + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == vdo->thread_config.admin_thread), + "%s called on admin thread", name); +} + +/** + * vdo_assert_on_logical_zone_thread() - Assert that this function was called on the specified + * logical zone thread. + * @vdo: The vdo. + * @logical_zone: The number of the logical zone. + * @name: The name of the calling function. + */ +void vdo_assert_on_logical_zone_thread(const struct vdo *vdo, zone_count_t logical_zone, + const char *name) +{ + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == + vdo->thread_config.logical_threads[logical_zone]), + "%s called on logical thread", name); +} + +/** + * vdo_assert_on_physical_zone_thread() - Assert that this function was called on the specified + * physical zone thread. + * @vdo: The vdo. + * @physical_zone: The number of the physical zone. + * @name: The name of the calling function. + */ +void vdo_assert_on_physical_zone_thread(const struct vdo *vdo, + zone_count_t physical_zone, const char *name) +{ + VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == + vdo->thread_config.physical_threads[physical_zone]), + "%s called on physical thread", name); +} + +/** + * vdo_get_physical_zone() - Get the physical zone responsible for a given physical block number. + * @vdo: The vdo containing the physical zones. + * @pbn: The PBN of the data block. + * @zone_ptr: A pointer to return the physical zone. + * + * Gets the physical zone responsible for a given physical block number of a data block in this vdo + * instance, or of the zero block (for which a NULL zone is returned). For any other block number + * that is not in the range of valid data block numbers in any slab, an error will be returned. + * This function is safe to call on invalid block numbers; it will not put the vdo into read-only + * mode. + * + * Return: VDO_SUCCESS or VDO_OUT_OF_RANGE if the block number is invalid or an error code for any + * other failure. + */ +int vdo_get_physical_zone(const struct vdo *vdo, physical_block_number_t pbn, + struct physical_zone **zone_ptr) +{ + struct vdo_slab *slab; + int result; + + if (pbn == VDO_ZERO_BLOCK) { + *zone_ptr = NULL; + return VDO_SUCCESS; + } + + /* + * Used because it does a more restrictive bounds check than vdo_get_slab(), and done first + * because it won't trigger read-only mode on an invalid PBN. + */ + if (!vdo_is_physical_data_block(vdo->depot, pbn)) + return VDO_OUT_OF_RANGE; + + /* With the PBN already checked, we should always succeed in finding a slab. */ + slab = vdo_get_slab(vdo->depot, pbn); + result = VDO_ASSERT(slab != NULL, "vdo_get_slab must succeed on all valid PBNs"); + if (result != VDO_SUCCESS) + return result; + + *zone_ptr = &vdo->physical_zones->zones[slab->allocator->zone_number]; + return VDO_SUCCESS; +} diff --git a/drivers/md/dm-vdo/vdo.h b/drivers/md/dm-vdo/vdo.h new file mode 100644 index 000000000000..483ae873e002 --- /dev/null +++ b/drivers/md/dm-vdo/vdo.h @@ -0,0 +1,362 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_H +#define VDO_H + +#include +#include +#include +#include +#include +#include + +#include "admin-state.h" +#include "encodings.h" +#include "funnel-workqueue.h" +#include "packer.h" +#include "physical-zone.h" +#include "statistics.h" +#include "thread-registry.h" +#include "types.h" + +enum notifier_state { + /* Notifications are allowed but not in progress */ + MAY_NOTIFY, + /* A notification is in progress */ + NOTIFYING, + /* Notifications are not allowed */ + MAY_NOT_NOTIFY, + /* A notification has completed */ + NOTIFIED, +}; + +/** + * typedef vdo_read_only_notification_fn - A function to notify a listener that the VDO has gone + * read-only. + * @listener: The object to notify. + * @parent: The completion to notify in order to acknowledge the notification. + */ +typedef void (*vdo_read_only_notification_fn)(void *listener, struct vdo_completion *parent); + +/* + * An object to be notified when the VDO enters read-only mode + */ +struct read_only_listener { + /* The listener */ + void *listener; + /* The method to call to notify the listener */ + vdo_read_only_notification_fn notify; + /* A pointer to the next listener */ + struct read_only_listener *next; +}; + +struct vdo_thread { + struct vdo *vdo; + thread_id_t thread_id; + struct vdo_work_queue *queue; + /* + * Each thread maintains its own notion of whether the VDO is read-only so that the + * read-only state can be checked from any base thread without worrying about + * synchronization or thread safety. This does mean that knowledge of the VDO going + * read-only does not occur simultaneously across the VDO's threads, but that does not seem + * to cause any problems. + */ + bool is_read_only; + /* + * A list of objects waiting to be notified on this thread that the VDO has entered + * read-only mode. + */ + struct read_only_listener *listeners; + struct registered_thread allocating_thread; +}; + +/* Keep struct bio statistics atomically */ +struct atomic_bio_stats { + atomic64_t read; /* Number of not REQ_WRITE bios */ + atomic64_t write; /* Number of REQ_WRITE bios */ + atomic64_t discard; /* Number of REQ_DISCARD bios */ + atomic64_t flush; /* Number of REQ_FLUSH bios */ + atomic64_t empty_flush; /* Number of REQ_PREFLUSH bios without data */ + atomic64_t fua; /* Number of REQ_FUA bios */ +}; + +/* Counters are atomic since updates can arrive concurrently from arbitrary threads. */ +struct atomic_statistics { + atomic64_t bios_submitted; + atomic64_t bios_completed; + atomic64_t flush_out; + atomic64_t invalid_advice_pbn_count; + atomic64_t no_space_error_count; + atomic64_t read_only_error_count; + struct atomic_bio_stats bios_in; + struct atomic_bio_stats bios_in_partial; + struct atomic_bio_stats bios_out; + struct atomic_bio_stats bios_out_completed; + struct atomic_bio_stats bios_acknowledged; + struct atomic_bio_stats bios_acknowledged_partial; + struct atomic_bio_stats bios_meta; + struct atomic_bio_stats bios_meta_completed; + struct atomic_bio_stats bios_journal; + struct atomic_bio_stats bios_journal_completed; + struct atomic_bio_stats bios_page_cache; + struct atomic_bio_stats bios_page_cache_completed; +}; + +struct read_only_notifier { + /* The completion for entering read-only mode */ + struct vdo_completion completion; + /* A completion waiting for notifications to be drained or enabled */ + struct vdo_completion *waiter; + /* Lock to protect the next two fields */ + spinlock_t lock; + /* The code of the error which put the VDO into read-only mode */ + int read_only_error; + /* The current state of the notifier (values described above) */ + enum notifier_state state; +}; + +/* + * The thread ID returned when the current thread is not a vdo thread, or can not be determined + * (usually due to being at interrupt context). + */ +#define VDO_INVALID_THREAD_ID ((thread_id_t) -1) + +struct thread_config { + zone_count_t logical_zone_count; + zone_count_t physical_zone_count; + zone_count_t hash_zone_count; + thread_count_t bio_thread_count; + thread_count_t thread_count; + thread_id_t admin_thread; + thread_id_t journal_thread; + thread_id_t packer_thread; + thread_id_t dedupe_thread; + thread_id_t bio_ack_thread; + thread_id_t cpu_thread; + thread_id_t *logical_threads; + thread_id_t *physical_threads; + thread_id_t *hash_zone_threads; + thread_id_t *bio_threads; +}; + +struct thread_count_config; + +struct vdo_super_block { + /* The vio for reading and writing the super block to disk */ + struct vio vio; + /* A buffer to hold the super block */ + u8 *buffer; + /* Whether this super block may not be written */ + bool unwritable; +}; + +struct data_vio_pool; + +struct vdo_administrator { + struct vdo_completion completion; + struct admin_state state; + atomic_t busy; + u32 phase; + struct completion callback_sync; +}; + +struct vdo { + char thread_name_prefix[MAX_VDO_WORK_QUEUE_NAME_LEN]; + struct vdo_thread *threads; + vdo_action_fn action; + struct vdo_completion *completion; + struct vio_tracer *vio_tracer; + + /* The atomic version of the state of this vdo */ + atomic_t state; + /* The full state of all components */ + struct vdo_component_states states; + /* + * A counter value to attach to thread names and log messages to identify the individual + * device. + */ + unsigned int instance; + /* The read-only notifier */ + struct read_only_notifier read_only_notifier; + /* The load-time configuration of this vdo */ + struct device_config *device_config; + /* The thread mapping */ + struct thread_config thread_config; + + /* The super block */ + struct vdo_super_block super_block; + + /* The partitioning of the underlying storage */ + struct layout layout; + struct layout next_layout; + struct dm_kcopyd_client *partition_copier; + + /* The block map */ + struct block_map *block_map; + + /* The journal for block map recovery */ + struct recovery_journal *recovery_journal; + + /* The slab depot */ + struct slab_depot *depot; + + /* The compressed-block packer */ + struct packer *packer; + /* Whether incoming data should be compressed */ + bool compressing; + + /* The handler for flush requests */ + struct flusher *flusher; + + /* The state the vdo was in when loaded (primarily for unit tests) */ + enum vdo_state load_state; + + /* The logical zones of this vdo */ + struct logical_zones *logical_zones; + + /* The physical zones of this vdo */ + struct physical_zones *physical_zones; + + /* The hash lock zones of this vdo */ + struct hash_zones *hash_zones; + + /* Bio submission manager used for sending bios to the storage device. */ + struct io_submitter *io_submitter; + + /* The pool of data_vios for servicing incoming bios */ + struct data_vio_pool *data_vio_pool; + + /* The manager for administrative operations */ + struct vdo_administrator admin; + + /* Flags controlling administrative operations */ + const struct admin_state_code *suspend_type; + bool allocations_allowed; + bool dump_on_shutdown; + atomic_t processing_message; + + /* + * Statistics + * Atomic stats counters + */ + struct atomic_statistics stats; + /* Used to gather statistics without allocating memory */ + struct vdo_statistics stats_buffer; + /* Protects the stats_buffer */ + struct mutex stats_mutex; + + /* A list of all device_configs referencing this vdo */ + struct list_head device_config_list; + + /* This VDO's list entry for the device registry */ + struct list_head registration; + + /* Underlying block device info. */ + u64 starting_sector_offset; + struct volume_geometry geometry; + + /* N blobs of context data for LZ4 code, one per CPU thread. */ + char **compression_context; +}; + +/** + * vdo_uses_bio_ack_queue() - Indicate whether the vdo is configured to use a separate work queue + * for acknowledging received and processed bios. + * @vdo: The vdo. + * + * Note that this directly controls the handling of write operations, but the compile-time flag + * VDO_USE_BIO_ACK_QUEUE_FOR_READ is also checked for read operations. + * + * Return: Whether a bio-acknowledgement work queue is in use. + */ +static inline bool vdo_uses_bio_ack_queue(struct vdo *vdo) +{ + return vdo->device_config->thread_counts.bio_ack_threads > 0; +} + +/** + * typedef vdo_filter_fn - Method type for vdo matching methods. + * + * A filter function returns false if the vdo doesn't match. + */ +typedef bool (*vdo_filter_fn)(struct vdo *vdo, const void *context); + +void vdo_initialize_device_registry_once(void); +struct vdo * __must_check vdo_find_matching(vdo_filter_fn filter, const void *context); + +int __must_check vdo_make_thread(struct vdo *vdo, thread_id_t thread_id, + const struct vdo_work_queue_type *type, + unsigned int queue_count, void *contexts[]); + +static inline int __must_check vdo_make_default_thread(struct vdo *vdo, + thread_id_t thread_id) +{ + return vdo_make_thread(vdo, thread_id, NULL, 1, NULL); +} + +int __must_check vdo_make(unsigned int instance, struct device_config *config, + char **reason, struct vdo **vdo_ptr); + +void vdo_destroy(struct vdo *vdo); + +void vdo_load_super_block(struct vdo *vdo, struct vdo_completion *parent); + +struct block_device * __must_check vdo_get_backing_device(const struct vdo *vdo); + +const char * __must_check vdo_get_device_name(const struct dm_target *target); + +int __must_check vdo_synchronous_flush(struct vdo *vdo); + +const struct admin_state_code * __must_check vdo_get_admin_state(const struct vdo *vdo); + +bool vdo_set_compressing(struct vdo *vdo, bool enable); + +bool vdo_get_compressing(struct vdo *vdo); + +void vdo_fetch_statistics(struct vdo *vdo, struct vdo_statistics *stats); + +thread_id_t vdo_get_callback_thread_id(void); + +enum vdo_state __must_check vdo_get_state(const struct vdo *vdo); + +void vdo_set_state(struct vdo *vdo, enum vdo_state state); + +void vdo_save_components(struct vdo *vdo, struct vdo_completion *parent); + +int vdo_register_read_only_listener(struct vdo *vdo, void *listener, + vdo_read_only_notification_fn notification, + thread_id_t thread_id); + +int vdo_enable_read_only_entry(struct vdo *vdo); + +void vdo_wait_until_not_entering_read_only_mode(struct vdo_completion *parent); + +void vdo_allow_read_only_mode_entry(struct vdo_completion *parent); + +void vdo_enter_read_only_mode(struct vdo *vdo, int error_code); + +bool __must_check vdo_is_read_only(struct vdo *vdo); + +bool __must_check vdo_in_read_only_mode(const struct vdo *vdo); + +bool __must_check vdo_in_recovery_mode(const struct vdo *vdo); + +void vdo_enter_recovery_mode(struct vdo *vdo); + +void vdo_assert_on_admin_thread(const struct vdo *vdo, const char *name); + +void vdo_assert_on_logical_zone_thread(const struct vdo *vdo, zone_count_t logical_zone, + const char *name); + +void vdo_assert_on_physical_zone_thread(const struct vdo *vdo, zone_count_t physical_zone, + const char *name); + +int __must_check vdo_get_physical_zone(const struct vdo *vdo, physical_block_number_t pbn, + struct physical_zone **zone_ptr); + +void vdo_dump_status(const struct vdo *vdo); + +#endif /* VDO_H */ diff --git a/drivers/md/dm-vdo/vio.c b/drivers/md/dm-vdo/vio.c new file mode 100644 index 000000000000..b291578f726f --- /dev/null +++ b/drivers/md/dm-vdo/vio.c @@ -0,0 +1,500 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "vio.h" + +#include +#include +#include +#include + +#include "logger.h" +#include "memory-alloc.h" +#include "permassert.h" + +#include "constants.h" +#include "io-submitter.h" +#include "vdo.h" + +/* A vio_pool is a collection of preallocated vios. */ +struct vio_pool { + /* The number of objects managed by the pool */ + size_t size; + /* The list of objects which are available */ + struct list_head available; + /* The queue of requestors waiting for objects from the pool */ + struct vdo_wait_queue waiting; + /* The number of objects currently in use */ + size_t busy_count; + /* The list of objects which are in use */ + struct list_head busy; + /* The ID of the thread on which this pool may be used */ + thread_id_t thread_id; + /* The buffer backing the pool's vios */ + char *buffer; + /* The pool entries */ + struct pooled_vio vios[]; +}; + +physical_block_number_t pbn_from_vio_bio(struct bio *bio) +{ + struct vio *vio = bio->bi_private; + struct vdo *vdo = vio->completion.vdo; + physical_block_number_t pbn = bio->bi_iter.bi_sector / VDO_SECTORS_PER_BLOCK; + + return ((pbn == VDO_GEOMETRY_BLOCK_LOCATION) ? pbn : pbn + vdo->geometry.bio_offset); +} + +static int create_multi_block_bio(block_count_t size, struct bio **bio_ptr) +{ + struct bio *bio = NULL; + int result; + + result = vdo_allocate_extended(struct bio, size + 1, struct bio_vec, + "bio", &bio); + if (result != VDO_SUCCESS) + return result; + + *bio_ptr = bio; + return VDO_SUCCESS; +} + +int vdo_create_bio(struct bio **bio_ptr) +{ + return create_multi_block_bio(1, bio_ptr); +} + +void vdo_free_bio(struct bio *bio) +{ + if (bio == NULL) + return; + + bio_uninit(bio); + vdo_free(vdo_forget(bio)); +} + +int allocate_vio_components(struct vdo *vdo, enum vio_type vio_type, + enum vio_priority priority, void *parent, + unsigned int block_count, char *data, struct vio *vio) +{ + struct bio *bio; + int result; + + result = VDO_ASSERT(block_count <= MAX_BLOCKS_PER_VIO, + "block count %u does not exceed maximum %u", block_count, + MAX_BLOCKS_PER_VIO); + if (result != VDO_SUCCESS) + return result; + + result = VDO_ASSERT(((vio_type != VIO_TYPE_UNINITIALIZED) && (vio_type != VIO_TYPE_DATA)), + "%d is a metadata type", vio_type); + if (result != VDO_SUCCESS) + return result; + + result = create_multi_block_bio(block_count, &bio); + if (result != VDO_SUCCESS) + return result; + + initialize_vio(vio, bio, block_count, vio_type, priority, vdo); + vio->completion.parent = parent; + vio->data = data; + return VDO_SUCCESS; +} + +/** + * create_multi_block_metadata_vio() - Create a vio. + * @vdo: The vdo on which the vio will operate. + * @vio_type: The type of vio to create. + * @priority: The relative priority to assign to the vio. + * @parent: The parent of the vio. + * @block_count: The size of the vio in blocks. + * @data: The buffer. + * @vio_ptr: A pointer to hold the new vio. + * + * Return: VDO_SUCCESS or an error. + */ +int create_multi_block_metadata_vio(struct vdo *vdo, enum vio_type vio_type, + enum vio_priority priority, void *parent, + unsigned int block_count, char *data, + struct vio **vio_ptr) +{ + struct vio *vio; + int result; + + BUILD_BUG_ON(sizeof(struct vio) > 256); + + /* + * Metadata vios should use direct allocation and not use the buffer pool, which is + * reserved for submissions from the linux block layer. + */ + result = vdo_allocate(1, struct vio, __func__, &vio); + if (result != VDO_SUCCESS) { + vdo_log_error("metadata vio allocation failure %d", result); + return result; + } + + result = allocate_vio_components(vdo, vio_type, priority, parent, block_count, + data, vio); + if (result != VDO_SUCCESS) { + vdo_free(vio); + return result; + } + + *vio_ptr = vio; + return VDO_SUCCESS; +} + +/** + * free_vio_components() - Free the components of a vio embedded in a larger structure. + * @vio: The vio to destroy + */ +void free_vio_components(struct vio *vio) +{ + if (vio == NULL) + return; + + BUG_ON(is_data_vio(vio)); + vdo_free_bio(vdo_forget(vio->bio)); +} + +/** + * free_vio() - Destroy a vio. + * @vio: The vio to destroy. + */ +void free_vio(struct vio *vio) +{ + free_vio_components(vio); + vdo_free(vio); +} + +/* Set bio properties for a VDO read or write. */ +void vdo_set_bio_properties(struct bio *bio, struct vio *vio, bio_end_io_t callback, + blk_opf_t bi_opf, physical_block_number_t pbn) +{ + struct vdo *vdo = vio->completion.vdo; + struct device_config *config = vdo->device_config; + + pbn -= vdo->geometry.bio_offset; + vio->bio_zone = ((pbn / config->thread_counts.bio_rotation_interval) % + config->thread_counts.bio_threads); + + bio->bi_private = vio; + bio->bi_end_io = callback; + bio->bi_opf = bi_opf; + bio->bi_iter.bi_sector = pbn * VDO_SECTORS_PER_BLOCK; +} + +/* + * Prepares the bio to perform IO with the specified buffer. May only be used on a VDO-allocated + * bio, as it assumes the bio wraps a 4k buffer that is 4k aligned, but there does not have to be a + * vio associated with the bio. + */ +int vio_reset_bio(struct vio *vio, char *data, bio_end_io_t callback, + blk_opf_t bi_opf, physical_block_number_t pbn) +{ + int bvec_count, offset, len, i; + struct bio *bio = vio->bio; + + bio_reset(bio, bio->bi_bdev, bi_opf); + vdo_set_bio_properties(bio, vio, callback, bi_opf, pbn); + if (data == NULL) + return VDO_SUCCESS; + + bio->bi_io_vec = bio->bi_inline_vecs; + bio->bi_max_vecs = vio->block_count + 1; + len = VDO_BLOCK_SIZE * vio->block_count; + offset = offset_in_page(data); + bvec_count = DIV_ROUND_UP(offset + len, PAGE_SIZE); + + /* + * If we knew that data was always on one page, or contiguous pages, we wouldn't need the + * loop. But if we're using vmalloc, it's not impossible that the data is in different + * pages that can't be merged in bio_add_page... + */ + for (i = 0; (i < bvec_count) && (len > 0); i++) { + struct page *page; + int bytes_added; + int bytes = PAGE_SIZE - offset; + + if (bytes > len) + bytes = len; + + page = is_vmalloc_addr(data) ? vmalloc_to_page(data) : virt_to_page(data); + bytes_added = bio_add_page(bio, page, bytes, offset); + + if (bytes_added != bytes) { + return vdo_log_error_strerror(VDO_BIO_CREATION_FAILED, + "Could only add %i bytes to bio", + bytes_added); + } + + data += bytes; + len -= bytes; + offset = 0; + } + + return VDO_SUCCESS; +} + +/** + * update_vio_error_stats() - Update per-vio error stats and log the error. + * @vio: The vio which got an error. + * @format: The format of the message to log (a printf style format). + */ +void update_vio_error_stats(struct vio *vio, const char *format, ...) +{ + static DEFINE_RATELIMIT_STATE(error_limiter, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + va_list args; + int priority; + struct vdo *vdo = vio->completion.vdo; + + switch (vio->completion.result) { + case VDO_READ_ONLY: + atomic64_inc(&vdo->stats.read_only_error_count); + return; + + case VDO_NO_SPACE: + atomic64_inc(&vdo->stats.no_space_error_count); + priority = VDO_LOG_DEBUG; + break; + + default: + priority = VDO_LOG_ERR; + } + + if (!__ratelimit(&error_limiter)) + return; + + va_start(args, format); + vdo_vlog_strerror(priority, vio->completion.result, VDO_LOGGING_MODULE_NAME, + format, args); + va_end(args); +} + +void vio_record_metadata_io_error(struct vio *vio) +{ + const char *description; + physical_block_number_t pbn = pbn_from_vio_bio(vio->bio); + + if (bio_op(vio->bio) == REQ_OP_READ) { + description = "read"; + } else if ((vio->bio->bi_opf & REQ_PREFLUSH) == REQ_PREFLUSH) { + description = (((vio->bio->bi_opf & REQ_FUA) == REQ_FUA) ? + "write+preflush+fua" : + "write+preflush"); + } else if ((vio->bio->bi_opf & REQ_FUA) == REQ_FUA) { + description = "write+fua"; + } else { + description = "write"; + } + + update_vio_error_stats(vio, + "Completing %s vio of type %u for physical block %llu with error", + description, vio->type, (unsigned long long) pbn); +} + +/** + * make_vio_pool() - Create a new vio pool. + * @vdo: The vdo. + * @pool_size: The number of vios in the pool. + * @thread_id: The ID of the thread using this pool. + * @vio_type: The type of vios in the pool. + * @priority: The priority with which vios from the pool should be enqueued. + * @context: The context that each entry will have. + * @pool_ptr: The resulting pool. + * + * Return: A success or error code. + */ +int make_vio_pool(struct vdo *vdo, size_t pool_size, thread_id_t thread_id, + enum vio_type vio_type, enum vio_priority priority, void *context, + struct vio_pool **pool_ptr) +{ + struct vio_pool *pool; + char *ptr; + int result; + + result = vdo_allocate_extended(struct vio_pool, pool_size, struct pooled_vio, + __func__, &pool); + if (result != VDO_SUCCESS) + return result; + + pool->thread_id = thread_id; + INIT_LIST_HEAD(&pool->available); + INIT_LIST_HEAD(&pool->busy); + + result = vdo_allocate(pool_size * VDO_BLOCK_SIZE, char, + "VIO pool buffer", &pool->buffer); + if (result != VDO_SUCCESS) { + free_vio_pool(pool); + return result; + } + + ptr = pool->buffer; + for (pool->size = 0; pool->size < pool_size; pool->size++, ptr += VDO_BLOCK_SIZE) { + struct pooled_vio *pooled = &pool->vios[pool->size]; + + result = allocate_vio_components(vdo, vio_type, priority, NULL, 1, ptr, + &pooled->vio); + if (result != VDO_SUCCESS) { + free_vio_pool(pool); + return result; + } + + pooled->context = context; + list_add_tail(&pooled->pool_entry, &pool->available); + } + + *pool_ptr = pool; + return VDO_SUCCESS; +} + +/** + * free_vio_pool() - Destroy a vio pool. + * @pool: The pool to free. + */ +void free_vio_pool(struct vio_pool *pool) +{ + struct pooled_vio *pooled, *tmp; + + if (pool == NULL) + return; + + /* Remove all available vios from the object pool. */ + VDO_ASSERT_LOG_ONLY(!vdo_waitq_has_waiters(&pool->waiting), + "VIO pool must not have any waiters when being freed"); + VDO_ASSERT_LOG_ONLY((pool->busy_count == 0), + "VIO pool must not have %zu busy entries when being freed", + pool->busy_count); + VDO_ASSERT_LOG_ONLY(list_empty(&pool->busy), + "VIO pool must not have busy entries when being freed"); + + list_for_each_entry_safe(pooled, tmp, &pool->available, pool_entry) { + list_del(&pooled->pool_entry); + free_vio_components(&pooled->vio); + pool->size--; + } + + VDO_ASSERT_LOG_ONLY(pool->size == 0, + "VIO pool must not have missing entries when being freed"); + + vdo_free(vdo_forget(pool->buffer)); + vdo_free(pool); +} + +/** + * is_vio_pool_busy() - Check whether an vio pool has outstanding entries. + * + * Return: true if the pool is busy. + */ +bool is_vio_pool_busy(struct vio_pool *pool) +{ + return (pool->busy_count != 0); +} + +/** + * acquire_vio_from_pool() - Acquire a vio and buffer from the pool (asynchronous). + * @pool: The vio pool. + * @waiter: Object that is requesting a vio. + */ +void acquire_vio_from_pool(struct vio_pool *pool, struct vdo_waiter *waiter) +{ + struct pooled_vio *pooled; + + VDO_ASSERT_LOG_ONLY((pool->thread_id == vdo_get_callback_thread_id()), + "acquire from active vio_pool called from correct thread"); + + if (list_empty(&pool->available)) { + vdo_waitq_enqueue_waiter(&pool->waiting, waiter); + return; + } + + pooled = list_first_entry(&pool->available, struct pooled_vio, pool_entry); + pool->busy_count++; + list_move_tail(&pooled->pool_entry, &pool->busy); + (*waiter->callback)(waiter, pooled); +} + +/** + * return_vio_to_pool() - Return a vio to the pool + * @pool: The vio pool. + * @vio: The pooled vio to return. + */ +void return_vio_to_pool(struct vio_pool *pool, struct pooled_vio *vio) +{ + VDO_ASSERT_LOG_ONLY((pool->thread_id == vdo_get_callback_thread_id()), + "vio pool entry returned on same thread as it was acquired"); + + vio->vio.completion.error_handler = NULL; + vio->vio.completion.parent = NULL; + if (vdo_waitq_has_waiters(&pool->waiting)) { + vdo_waitq_notify_next_waiter(&pool->waiting, NULL, vio); + return; + } + + list_move_tail(&vio->pool_entry, &pool->available); + --pool->busy_count; +} + +/* + * Various counting functions for statistics. + * These are used for bios coming into VDO, as well as bios generated by VDO. + */ +void vdo_count_bios(struct atomic_bio_stats *bio_stats, struct bio *bio) +{ + if (((bio->bi_opf & REQ_PREFLUSH) != 0) && (bio->bi_iter.bi_size == 0)) { + atomic64_inc(&bio_stats->empty_flush); + atomic64_inc(&bio_stats->flush); + return; + } + + switch (bio_op(bio)) { + case REQ_OP_WRITE: + atomic64_inc(&bio_stats->write); + break; + case REQ_OP_READ: + atomic64_inc(&bio_stats->read); + break; + case REQ_OP_DISCARD: + atomic64_inc(&bio_stats->discard); + break; + /* + * All other operations are filtered out in dmvdo.c, or not created by VDO, so + * shouldn't exist. + */ + default: + VDO_ASSERT_LOG_ONLY(0, "Bio operation %d not a write, read, discard, or empty flush", + bio_op(bio)); + } + + if ((bio->bi_opf & REQ_PREFLUSH) != 0) + atomic64_inc(&bio_stats->flush); + if (bio->bi_opf & REQ_FUA) + atomic64_inc(&bio_stats->fua); +} + +static void count_all_bios_completed(struct vio *vio, struct bio *bio) +{ + struct atomic_statistics *stats = &vio->completion.vdo->stats; + + if (is_data_vio(vio)) { + vdo_count_bios(&stats->bios_out_completed, bio); + return; + } + + vdo_count_bios(&stats->bios_meta_completed, bio); + if (vio->type == VIO_TYPE_RECOVERY_JOURNAL) + vdo_count_bios(&stats->bios_journal_completed, bio); + else if (vio->type == VIO_TYPE_BLOCK_MAP) + vdo_count_bios(&stats->bios_page_cache_completed, bio); +} + +void vdo_count_completed_bios(struct bio *bio) +{ + struct vio *vio = (struct vio *) bio->bi_private; + + atomic64_inc(&vio->completion.vdo->stats.bios_completed); + count_all_bios_completed(vio, bio); +} diff --git a/drivers/md/dm-vdo/vio.h b/drivers/md/dm-vdo/vio.h new file mode 100644 index 000000000000..3490e9f59b04 --- /dev/null +++ b/drivers/md/dm-vdo/vio.h @@ -0,0 +1,199 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VIO_H +#define VIO_H + +#include +#include +#include +#include +#include + +#include "completion.h" +#include "constants.h" +#include "types.h" +#include "vdo.h" + +enum { + MAX_BLOCKS_PER_VIO = (BIO_MAX_VECS << PAGE_SHIFT) / VDO_BLOCK_SIZE, +}; + +struct pooled_vio { + /* The underlying vio */ + struct vio vio; + /* The list entry for chaining pooled vios together */ + struct list_head list_entry; + /* The context set by the pool */ + void *context; + /* The list entry used by the pool */ + struct list_head pool_entry; +}; + +/** + * as_vio() - Convert a generic vdo_completion to a vio. + * @completion: The completion to convert. + * + * Return: The completion as a vio. + */ +static inline struct vio *as_vio(struct vdo_completion *completion) +{ + vdo_assert_completion_type(completion, VIO_COMPLETION); + return container_of(completion, struct vio, completion); +} + +/** + * get_vio_bio_zone_thread_id() - Get the thread id of the bio zone in which a vio should submit + * its I/O. + * @vio: The vio. + * + * Return: The id of the bio zone thread the vio should use. + */ +static inline thread_id_t __must_check get_vio_bio_zone_thread_id(struct vio *vio) +{ + return vio->completion.vdo->thread_config.bio_threads[vio->bio_zone]; +} + +physical_block_number_t __must_check pbn_from_vio_bio(struct bio *bio); + +/** + * assert_vio_in_bio_zone() - Check that a vio is running on the correct thread for its bio zone. + * @vio: The vio to check. + */ +static inline void assert_vio_in_bio_zone(struct vio *vio) +{ + thread_id_t expected = get_vio_bio_zone_thread_id(vio); + thread_id_t thread_id = vdo_get_callback_thread_id(); + + VDO_ASSERT_LOG_ONLY((expected == thread_id), + "vio I/O for physical block %llu on thread %u, should be on bio zone thread %u", + (unsigned long long) pbn_from_vio_bio(vio->bio), thread_id, + expected); +} + +int vdo_create_bio(struct bio **bio_ptr); +void vdo_free_bio(struct bio *bio); +int allocate_vio_components(struct vdo *vdo, enum vio_type vio_type, + enum vio_priority priority, void *parent, + unsigned int block_count, char *data, struct vio *vio); +int __must_check create_multi_block_metadata_vio(struct vdo *vdo, enum vio_type vio_type, + enum vio_priority priority, + void *parent, unsigned int block_count, + char *data, struct vio **vio_ptr); + +static inline int __must_check create_metadata_vio(struct vdo *vdo, enum vio_type vio_type, + enum vio_priority priority, + void *parent, char *data, + struct vio **vio_ptr) +{ + return create_multi_block_metadata_vio(vdo, vio_type, priority, parent, 1, data, + vio_ptr); +} + +void free_vio_components(struct vio *vio); +void free_vio(struct vio *vio); + +/** + * initialize_vio() - Initialize a vio. + * @vio: The vio to initialize. + * @bio: The bio this vio should use for its I/O. + * @block_count: The size of this vio in vdo blocks. + * @vio_type: The vio type. + * @priority: The relative priority of the vio. + * @vdo: The vdo for this vio. + */ +static inline void initialize_vio(struct vio *vio, struct bio *bio, + unsigned int block_count, enum vio_type vio_type, + enum vio_priority priority, struct vdo *vdo) +{ + /* data_vio's may not span multiple blocks */ + BUG_ON((vio_type == VIO_TYPE_DATA) && (block_count != 1)); + + vio->bio = bio; + vio->block_count = block_count; + vio->type = vio_type; + vio->priority = priority; + vdo_initialize_completion(&vio->completion, vdo, VIO_COMPLETION); +} + +void vdo_set_bio_properties(struct bio *bio, struct vio *vio, bio_end_io_t callback, + blk_opf_t bi_opf, physical_block_number_t pbn); + +int vio_reset_bio(struct vio *vio, char *data, bio_end_io_t callback, + blk_opf_t bi_opf, physical_block_number_t pbn); + +void update_vio_error_stats(struct vio *vio, const char *format, ...) + __printf(2, 3); + +/** + * is_data_vio() - Check whether a vio is servicing an external data request. + * @vio: The vio to check. + */ +static inline bool is_data_vio(struct vio *vio) +{ + return (vio->type == VIO_TYPE_DATA); +} + +/** + * get_metadata_priority() - Convert a vio's priority to a work item priority. + * @vio: The vio. + * + * Return: The priority with which to submit the vio's bio. + */ +static inline enum vdo_completion_priority get_metadata_priority(struct vio *vio) +{ + return ((vio->priority == VIO_PRIORITY_HIGH) ? + BIO_Q_HIGH_PRIORITY : + BIO_Q_METADATA_PRIORITY); +} + +/** + * continue_vio() - Enqueue a vio to run its next callback. + * @vio: The vio to continue. + * + * Return: The result of the current operation. + */ +static inline void continue_vio(struct vio *vio, int result) +{ + if (unlikely(result != VDO_SUCCESS)) + vdo_set_completion_result(&vio->completion, result); + + vdo_enqueue_completion(&vio->completion, VDO_WORK_Q_DEFAULT_PRIORITY); +} + +void vdo_count_bios(struct atomic_bio_stats *bio_stats, struct bio *bio); +void vdo_count_completed_bios(struct bio *bio); + +/** + * continue_vio_after_io() - Continue a vio now that its I/O has returned. + */ +static inline void continue_vio_after_io(struct vio *vio, vdo_action_fn callback, + thread_id_t thread) +{ + vdo_count_completed_bios(vio->bio); + vdo_set_completion_callback(&vio->completion, callback, thread); + continue_vio(vio, blk_status_to_errno(vio->bio->bi_status)); +} + +void vio_record_metadata_io_error(struct vio *vio); + +/* A vio_pool is a collection of preallocated vios used to write arbitrary metadata blocks. */ + +static inline struct pooled_vio *vio_as_pooled_vio(struct vio *vio) +{ + return container_of(vio, struct pooled_vio, vio); +} + +struct vio_pool; + +int __must_check make_vio_pool(struct vdo *vdo, size_t pool_size, thread_id_t thread_id, + enum vio_type vio_type, enum vio_priority priority, + void *context, struct vio_pool **pool_ptr); +void free_vio_pool(struct vio_pool *pool); +bool __must_check is_vio_pool_busy(struct vio_pool *pool); +void acquire_vio_from_pool(struct vio_pool *pool, struct vdo_waiter *waiter); +void return_vio_to_pool(struct vio_pool *pool, struct pooled_vio *vio); + +#endif /* VIO_H */ diff --git a/drivers/md/dm-vdo/wait-queue.c b/drivers/md/dm-vdo/wait-queue.c new file mode 100644 index 000000000000..6e1e739277ef --- /dev/null +++ b/drivers/md/dm-vdo/wait-queue.c @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Red Hat + */ + +#include "wait-queue.h" + +#include + +#include "permassert.h" + +#include "status-codes.h" + +/** + * vdo_waitq_enqueue_waiter() - Add a waiter to the tail end of a waitq. + * @waitq: The vdo_wait_queue to which to add the waiter. + * @waiter: The waiter to add to the waitq. + * + * The waiter must not already be waiting in a waitq. + */ +void vdo_waitq_enqueue_waiter(struct vdo_wait_queue *waitq, struct vdo_waiter *waiter) +{ + BUG_ON(waiter->next_waiter != NULL); + + if (waitq->last_waiter == NULL) { + /* + * The waitq is empty, so form the initial circular list by self-linking the + * initial waiter. + */ + waiter->next_waiter = waiter; + } else { + /* Splice the new waiter in at the end of the waitq. */ + waiter->next_waiter = waitq->last_waiter->next_waiter; + waitq->last_waiter->next_waiter = waiter; + } + + /* In both cases, the waiter we added to the ring becomes the last waiter. */ + waitq->last_waiter = waiter; + waitq->length += 1; +} + +/** + * vdo_waitq_transfer_all_waiters() - Transfer all waiters from one waitq to + * a second waitq, emptying the first waitq. + * @from_waitq: The waitq containing the waiters to move. + * @to_waitq: The waitq that will receive the waiters from the first waitq. + */ +void vdo_waitq_transfer_all_waiters(struct vdo_wait_queue *from_waitq, + struct vdo_wait_queue *to_waitq) +{ + /* If the source waitq is empty, there's nothing to do. */ + if (!vdo_waitq_has_waiters(from_waitq)) + return; + + if (vdo_waitq_has_waiters(to_waitq)) { + /* + * Both are non-empty. Splice the two circular lists together + * by swapping the next (head) pointers in the list tails. + */ + struct vdo_waiter *from_head = from_waitq->last_waiter->next_waiter; + struct vdo_waiter *to_head = to_waitq->last_waiter->next_waiter; + + to_waitq->last_waiter->next_waiter = from_head; + from_waitq->last_waiter->next_waiter = to_head; + } + + to_waitq->last_waiter = from_waitq->last_waiter; + to_waitq->length += from_waitq->length; + vdo_waitq_init(from_waitq); +} + +/** + * vdo_waitq_notify_all_waiters() - Notify all the entries waiting in a waitq. + * @waitq: The vdo_wait_queue containing the waiters to notify. + * @callback: The function to call to notify each waiter, or NULL to invoke the callback field + * registered in each waiter. + * @context: The context to pass to the callback function. + * + * Notifies all the entries waiting in a waitq to continue execution by invoking a callback + * function on each of them in turn. The waitq is copied and emptied before invoking any callbacks, + * and only the waiters that were in the waitq at the start of the call will be notified. + */ +void vdo_waitq_notify_all_waiters(struct vdo_wait_queue *waitq, + vdo_waiter_callback_fn callback, void *context) +{ + /* + * Copy and empty the waitq first, avoiding the possibility of an infinite + * loop if entries are returned to the waitq by the callback function. + */ + struct vdo_wait_queue waiters; + + vdo_waitq_init(&waiters); + vdo_waitq_transfer_all_waiters(waitq, &waiters); + + /* Drain the copied waitq, invoking the callback on every entry. */ + while (vdo_waitq_has_waiters(&waiters)) + vdo_waitq_notify_next_waiter(&waiters, callback, context); +} + +/** + * vdo_waitq_get_first_waiter() - Return the waiter that is at the head end of a waitq. + * @waitq: The vdo_wait_queue from which to get the first waiter. + * + * Return: The first (oldest) waiter in the waitq, or NULL if the waitq is empty. + */ +struct vdo_waiter *vdo_waitq_get_first_waiter(const struct vdo_wait_queue *waitq) +{ + struct vdo_waiter *last_waiter = waitq->last_waiter; + + if (last_waiter == NULL) { + /* There are no waiters, so we're done. */ + return NULL; + } + + /* The waitq is circular, so the last entry links to the head of the waitq. */ + return last_waiter->next_waiter; +} + +/** + * vdo_waitq_dequeue_matching_waiters() - Remove all waiters that match based on the specified + * matching method and append them to a vdo_wait_queue. + * @waitq: The vdo_wait_queue to process. + * @waiter_match: The method to determine matching. + * @match_context: Contextual info for the match method. + * @matched_waitq: A wait_waitq to store matches. + */ +void vdo_waitq_dequeue_matching_waiters(struct vdo_wait_queue *waitq, + vdo_waiter_match_fn waiter_match, + void *match_context, + struct vdo_wait_queue *matched_waitq) +{ + struct vdo_wait_queue iteration_waitq; + + vdo_waitq_init(&iteration_waitq); + vdo_waitq_transfer_all_waiters(waitq, &iteration_waitq); + + while (vdo_waitq_has_waiters(&iteration_waitq)) { + struct vdo_waiter *waiter = vdo_waitq_dequeue_waiter(&iteration_waitq); + + vdo_waitq_enqueue_waiter((waiter_match(waiter, match_context) ? + matched_waitq : waitq), waiter); + } +} + +/** + * vdo_waitq_dequeue_waiter() - Remove the first (oldest) waiter from a waitq. + * @waitq: The vdo_wait_queue from which to remove the first entry. + * + * The caller will be responsible for waking the waiter by continuing its + * execution appropriately. + * + * Return: The first (oldest) waiter in the waitq, or NULL if the waitq is empty. + */ +struct vdo_waiter *vdo_waitq_dequeue_waiter(struct vdo_wait_queue *waitq) +{ + struct vdo_waiter *first_waiter = vdo_waitq_get_first_waiter(waitq); + struct vdo_waiter *last_waiter = waitq->last_waiter; + + if (first_waiter == NULL) + return NULL; + + if (first_waiter == last_waiter) { + /* The waitq has a single entry, so empty it by nulling the tail. */ + waitq->last_waiter = NULL; + } else { + /* + * The waitq has multiple waiters, so splice the first waiter out + * of the circular waitq. + */ + last_waiter->next_waiter = first_waiter->next_waiter; + } + + /* The waiter is no longer in a waitq. */ + first_waiter->next_waiter = NULL; + waitq->length -= 1; + + return first_waiter; +} + +/** + * vdo_waitq_notify_next_waiter() - Notify the next entry waiting in a waitq. + * @waitq: The vdo_wait_queue containing the waiter to notify. + * @callback: The function to call to notify the waiter, or NULL to invoke the callback field + * registered in the waiter. + * @context: The context to pass to the callback function. + * + * Notifies the next entry waiting in a waitq to continue execution by invoking a callback function + * on it after removing it from the waitq. + * + * Return: true if there was a waiter in the waitq. + */ +bool vdo_waitq_notify_next_waiter(struct vdo_wait_queue *waitq, + vdo_waiter_callback_fn callback, void *context) +{ + struct vdo_waiter *waiter = vdo_waitq_dequeue_waiter(waitq); + + if (waiter == NULL) + return false; + + if (callback == NULL) + callback = waiter->callback; + callback(waiter, context); + + return true; +} diff --git a/drivers/md/dm-vdo/wait-queue.h b/drivers/md/dm-vdo/wait-queue.h new file mode 100644 index 000000000000..7e8ee6afe7c7 --- /dev/null +++ b/drivers/md/dm-vdo/wait-queue.h @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2023 Red Hat + */ + +#ifndef VDO_WAIT_QUEUE_H +#define VDO_WAIT_QUEUE_H + +#include +#include + +/** + * A vdo_wait_queue is a circular singly linked list of entries waiting to be notified + * of a change in a condition. Keeping a circular list allows the vdo_wait_queue + * structure to simply be a pointer to the tail (newest) entry, supporting + * constant-time enqueue and dequeue operations. A null pointer is an empty waitq. + * + * An empty waitq: + * waitq0.last_waiter -> NULL + * + * A singleton waitq: + * waitq1.last_waiter -> entry1 -> entry1 -> [...] + * + * A three-element waitq: + * waitq2.last_waiter -> entry3 -> entry1 -> entry2 -> entry3 -> [...] + * + * linux/wait.h's wait_queue_head is _not_ used because vdo_wait_queue's + * interface is much less complex (doesn't need locking, priorities or timers). + * Made possible by vdo's thread-based resource allocation and locking; and + * the polling nature of vdo_wait_queue consumers. + * + * FIXME: could be made to use a linux/list.h's list_head but its extra barriers + * really aren't needed. Nor is a doubly linked list, but vdo_wait_queue could + * make use of __list_del_clearprev() -- but that would compromise the ability + * to make full use of linux's list interface. + */ + +struct vdo_waiter; + +struct vdo_wait_queue { + /* The tail of the queue, the last (most recently added) entry */ + struct vdo_waiter *last_waiter; + /* The number of waiters currently in the queue */ + size_t length; +}; + +/** + * vdo_waiter_callback_fn - Callback type that will be called to resume processing + * of a waiter after it has been removed from its wait queue. + */ +typedef void (*vdo_waiter_callback_fn)(struct vdo_waiter *waiter, void *context); + +/** + * vdo_waiter_match_fn - Method type for waiter matching methods. + * + * Returns false if the waiter does not match. + */ +typedef bool (*vdo_waiter_match_fn)(struct vdo_waiter *waiter, void *context); + +/* The structure for entries in a vdo_wait_queue. */ +struct vdo_waiter { + /* + * The next waiter in the waitq. If this entry is the last waiter, then this + * is actually a pointer back to the head of the waitq. + */ + struct vdo_waiter *next_waiter; + + /* Optional waiter-specific callback to invoke when dequeuing this waiter. */ + vdo_waiter_callback_fn callback; +}; + +/** + * vdo_waiter_is_waiting() - Check whether a waiter is waiting. + * @waiter: The waiter to check. + * + * Return: true if the waiter is on some vdo_wait_queue. + */ +static inline bool vdo_waiter_is_waiting(struct vdo_waiter *waiter) +{ + return (waiter->next_waiter != NULL); +} + +/** + * vdo_waitq_init() - Initialize a vdo_wait_queue. + * @waitq: The vdo_wait_queue to initialize. + */ +static inline void vdo_waitq_init(struct vdo_wait_queue *waitq) +{ + *waitq = (struct vdo_wait_queue) { + .last_waiter = NULL, + .length = 0, + }; +} + +/** + * vdo_waitq_has_waiters() - Check whether a vdo_wait_queue has any entries waiting. + * @waitq: The vdo_wait_queue to query. + * + * Return: true if there are any waiters in the waitq. + */ +static inline bool __must_check vdo_waitq_has_waiters(const struct vdo_wait_queue *waitq) +{ + return (waitq->last_waiter != NULL); +} + +void vdo_waitq_enqueue_waiter(struct vdo_wait_queue *waitq, + struct vdo_waiter *waiter); + +struct vdo_waiter *vdo_waitq_dequeue_waiter(struct vdo_wait_queue *waitq); + +void vdo_waitq_notify_all_waiters(struct vdo_wait_queue *waitq, + vdo_waiter_callback_fn callback, void *context); + +bool vdo_waitq_notify_next_waiter(struct vdo_wait_queue *waitq, + vdo_waiter_callback_fn callback, void *context); + +void vdo_waitq_transfer_all_waiters(struct vdo_wait_queue *from_waitq, + struct vdo_wait_queue *to_waitq); + +struct vdo_waiter *vdo_waitq_get_first_waiter(const struct vdo_wait_queue *waitq); + +void vdo_waitq_dequeue_matching_waiters(struct vdo_wait_queue *waitq, + vdo_waiter_match_fn waiter_match, + void *match_context, + struct vdo_wait_queue *matched_waitq); + +/** + * vdo_waitq_num_waiters() - Return the number of waiters in a vdo_wait_queue. + * @waitq: The vdo_wait_queue to query. + * + * Return: The number of waiters in the waitq. + */ +static inline size_t __must_check vdo_waitq_num_waiters(const struct vdo_wait_queue *waitq) +{ + return waitq->length; +} + +#endif /* VDO_WAIT_QUEUE_H */ diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c index b475200d8586..e46aee6f932e 100644 --- a/drivers/md/dm-verity-fec.c +++ b/drivers/md/dm-verity-fec.c @@ -60,7 +60,8 @@ static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio, * to the data block. Caller is responsible for releasing buf. */ static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, - unsigned int *offset, struct dm_buffer **buf) + unsigned int *offset, struct dm_buffer **buf, + unsigned short ioprio) { u64 position, block, rem; u8 *res; @@ -69,7 +70,7 @@ static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, block = div64_u64_rem(position, v->fec->io_size, &rem); *offset = (unsigned int)rem; - res = dm_bufio_read(v->fec->bufio, block, buf); + res = dm_bufio_read_with_ioprio(v->fec->bufio, block, buf, ioprio); if (IS_ERR(res)) { DMERR("%s: FEC %llu: parity read failed (block %llu): %ld", v->data_dev->name, (unsigned long long)rsb, @@ -121,16 +122,17 @@ static inline unsigned int fec_buffer_rs_index(unsigned int i, unsigned int j) * Decode all RS blocks from buffers and copy corrected bytes into fio->output * starting from block_offset. */ -static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, - u64 rsb, int byte_index, unsigned int block_offset, - int neras) +static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_io *io, + struct dm_verity_fec_io *fio, u64 rsb, int byte_index, + unsigned int block_offset, int neras) { int r, corrected = 0, res; struct dm_buffer *buf; unsigned int n, i, offset; u8 *par, *block; + struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); - par = fec_read_parity(v, rsb, block_offset, &offset, &buf); + par = fec_read_parity(v, rsb, block_offset, &offset, &buf, bio_prio(bio)); if (IS_ERR(par)) return PTR_ERR(par); @@ -158,7 +160,7 @@ static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, if (offset >= v->fec->io_size) { dm_bufio_release(buf); - par = fec_read_parity(v, rsb, block_offset, &offset, &buf); + par = fec_read_parity(v, rsb, block_offset, &offset, &buf, bio_prio(bio)); if (IS_ERR(par)) return PTR_ERR(par); } @@ -210,6 +212,7 @@ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, u8 *bbuf, *rs_block; u8 want_digest[HASH_MAX_DIGESTSIZE]; unsigned int n, k; + struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); if (neras) *neras = 0; @@ -248,7 +251,7 @@ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, bufio = v->bufio; } - bbuf = dm_bufio_read(bufio, block, &buf); + bbuf = dm_bufio_read_with_ioprio(bufio, block, &buf, bio_prio(bio)); if (IS_ERR(bbuf)) { DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld", v->data_dev->name, @@ -377,7 +380,7 @@ static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, if (unlikely(r < 0)) return r; - r = fec_decode_bufs(v, fio, rsb, r, pos, neras); + r = fec_decode_bufs(v, io, fio, rsb, r, pos, neras); if (r < 0) return r; diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index 1b591bfa90d5..bb5da66da4c1 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -46,11 +46,12 @@ static unsigned int dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, 0644); -static DEFINE_STATIC_KEY_FALSE(use_tasklet_enabled); +static DEFINE_STATIC_KEY_FALSE(use_bh_wq_enabled); struct dm_verity_prefetch_work { struct work_struct work; struct dm_verity *v; + unsigned short ioprio; sector_t block; unsigned int n_blocks; }; @@ -294,10 +295,11 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, int r; sector_t hash_block; unsigned int offset; + struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); verity_hash_at_level(v, block, level, &hash_block, &offset); - if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) { + if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { data = dm_bufio_get(v->bufio, hash_block, &buf); if (data == NULL) { /* @@ -307,8 +309,10 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, */ return -EAGAIN; } - } else - data = dm_bufio_read(v->bufio, hash_block, &buf); + } else { + data = dm_bufio_read_with_ioprio(v->bufio, hash_block, + &buf, bio_prio(bio)); + } if (IS_ERR(data)) return PTR_ERR(data); @@ -323,15 +327,14 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, r = verity_hash(v, verity_io_hash_req(v, io), data, 1 << v->hash_dev_block_bits, - verity_io_real_digest(v, io), !io->in_tasklet); + verity_io_real_digest(v, io), !io->in_bh); if (unlikely(r < 0)) goto release_ret_r; if (likely(memcmp(verity_io_real_digest(v, io), want_digest, v->digest_size) == 0)) aux->hash_verified = 1; - else if (static_branch_unlikely(&use_tasklet_enabled) && - io->in_tasklet) { + else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { /* * Error handling code (FEC included) cannot be run in a * tasklet since it may sleep, so fallback to work-queue. @@ -511,7 +514,7 @@ static noinline int verity_recheck(struct dm_verity *v, struct dm_verity_io *io, io_loc.bdev = v->data_dev->bdev; io_loc.sector = cur_block << (v->data_dev_block_bits - SECTOR_SHIFT); io_loc.count = 1 << (v->data_dev_block_bits - SECTOR_SHIFT); - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) goto free_ret; @@ -572,7 +575,7 @@ static int verity_verify_io(struct dm_verity_io *io) struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); unsigned int b; - if (static_branch_unlikely(&use_tasklet_enabled) && io->in_tasklet) { + if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { /* * Copy the iterator in case we need to restart * verification in a work-queue. @@ -612,7 +615,7 @@ static int verity_verify_io(struct dm_verity_io *io) continue; } - r = verity_hash_init(v, req, &wait, !io->in_tasklet); + r = verity_hash_init(v, req, &wait, !io->in_bh); if (unlikely(r < 0)) return r; @@ -631,8 +634,7 @@ static int verity_verify_io(struct dm_verity_io *io) if (v->validated_blocks) set_bit(cur_block, v->validated_blocks); continue; - } else if (static_branch_unlikely(&use_tasklet_enabled) && - io->in_tasklet) { + } else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { /* * Error handling code (FEC included) cannot be run in a * tasklet since it may sleep, so fallback to work-queue. @@ -686,7 +688,7 @@ static void verity_finish_io(struct dm_verity_io *io, blk_status_t status) bio->bi_end_io = io->orig_bi_end_io; bio->bi_status = status; - if (!static_branch_unlikely(&use_tasklet_enabled) || !io->in_tasklet) + if (!static_branch_unlikely(&use_bh_wq_enabled) || !io->in_bh) verity_fec_finish_io(io); bio_endio(bio); @@ -696,11 +698,28 @@ static void verity_work(struct work_struct *w) { struct dm_verity_io *io = container_of(w, struct dm_verity_io, work); - io->in_tasklet = false; + io->in_bh = false; verity_finish_io(io, errno_to_blk_status(verity_verify_io(io))); } +static void verity_bh_work(struct work_struct *w) +{ + struct dm_verity_io *io = container_of(w, struct dm_verity_io, bh_work); + int err; + + io->in_bh = true; + err = verity_verify_io(io); + if (err == -EAGAIN || err == -ENOMEM) { + /* fallback to retrying with work-queue */ + INIT_WORK(&io->work, verity_work); + queue_work(io->v->verify_wq, &io->work); + return; + } + + verity_finish_io(io, errno_to_blk_status(err)); +} + static void verity_end_io(struct bio *bio) { struct dm_verity_io *io = bio->bi_private; @@ -713,8 +732,13 @@ static void verity_end_io(struct bio *bio) return; } - INIT_WORK(&io->work, verity_work); - queue_work(io->v->verify_wq, &io->work); + if (static_branch_unlikely(&use_bh_wq_enabled) && io->v->use_bh_wq) { + INIT_WORK(&io->bh_work, verity_bh_work); + queue_work(system_bh_wq, &io->bh_work); + } else { + INIT_WORK(&io->work, verity_work); + queue_work(io->v->verify_wq, &io->work); + } } /* @@ -752,14 +776,16 @@ static void verity_prefetch_io(struct work_struct *work) hash_block_end = v->hash_blocks - 1; } no_prefetch_cluster: - dm_bufio_prefetch(v->bufio, hash_block_start, - hash_block_end - hash_block_start + 1); + dm_bufio_prefetch_with_ioprio(v->bufio, hash_block_start, + hash_block_end - hash_block_start + 1, + pw->ioprio); } kfree(pw); } -static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io) +static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io, + unsigned short ioprio) { sector_t block = io->block; unsigned int n_blocks = io->n_blocks; @@ -787,6 +813,7 @@ static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io) pw->v = v; pw->block = block; pw->n_blocks = n_blocks; + pw->ioprio = ioprio; queue_work(v->verify_wq, &pw->work); } @@ -829,7 +856,7 @@ static int verity_map(struct dm_target *ti, struct bio *bio) verity_fec_init_io(io); - verity_submit_prefetch(v, io); + verity_submit_prefetch(v, io, bio_prio(bio)); submit_bio_noacct(bio); @@ -878,7 +905,7 @@ static void verity_status(struct dm_target *ti, status_type_t type, args++; if (v->validated_blocks) args++; - if (v->use_tasklet) + if (v->use_bh_wq) args++; if (v->signature_key_desc) args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS; @@ -905,7 +932,7 @@ static void verity_status(struct dm_target *ti, status_type_t type, DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES); if (v->validated_blocks) DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE); - if (v->use_tasklet) + if (v->use_bh_wq) DMEMIT(" " DM_VERITY_OPT_TASKLET_VERIFY); sz = verity_fec_status_table(v, sz, result, maxlen); if (v->signature_key_desc) @@ -1024,8 +1051,8 @@ static void verity_dtr(struct dm_target *ti) kfree(v->signature_key_desc); - if (v->use_tasklet) - static_branch_dec(&use_tasklet_enabled); + if (v->use_bh_wq) + static_branch_dec(&use_bh_wq_enabled); kfree(v); @@ -1159,8 +1186,8 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, continue; } else if (!strcasecmp(arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) { - v->use_tasklet = true; - static_branch_inc(&use_tasklet_enabled); + v->use_bh_wq = true; + static_branch_inc(&use_bh_wq_enabled); continue; } else if (verity_is_fec_opt_arg(arg_name)) { @@ -1331,7 +1358,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) } v->tfm = crypto_alloc_ahash(v->alg_name, 0, - v->use_tasklet ? CRYPTO_ALG_ASYNC : 0); + v->use_bh_wq ? CRYPTO_ALG_ASYNC : 0); if (IS_ERR(v->tfm)) { ti->error = "Cannot initialize hash function"; r = PTR_ERR(v->tfm); @@ -1456,7 +1483,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) v->bufio = dm_bufio_client_create(v->hash_dev->bdev, 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux), dm_bufio_alloc_callback, NULL, - v->use_tasklet ? DM_BUFIO_CLIENT_NO_SLEEP : 0); + v->use_bh_wq ? DM_BUFIO_CLIENT_NO_SLEEP : 0); if (IS_ERR(v->bufio)) { ti->error = "Cannot initialize dm-bufio"; r = PTR_ERR(v->bufio); @@ -1475,7 +1502,7 @@ static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) * reducing wait times when reading from a dm-verity device. * * Also as required for the "try_verify_in_tasklet" feature: WQ_HIGHPRI - * allows verify_wq to preempt softirq since verification in tasklet + * allows verify_wq to preempt softirq since verification in BH workqueue * will fall-back to using it for error handling (or if the bufio cache * doesn't have required hashes). */ @@ -1559,7 +1586,7 @@ int dm_verity_get_root_digest(struct dm_target *ti, u8 **root_digest, unsigned i static struct target_type verity_target = { .name = "verity", - .features = DM_TARGET_IMMUTABLE, + .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE, .version = {1, 10, 0}, .module = THIS_MODULE, .ctr = verity_ctr, diff --git a/drivers/md/dm-verity.h b/drivers/md/dm-verity.h index db93a91169d5..20b1bcf03474 100644 --- a/drivers/md/dm-verity.h +++ b/drivers/md/dm-verity.h @@ -54,7 +54,7 @@ struct dm_verity { unsigned char levels; /* the number of tree levels */ unsigned char version; bool hash_failed:1; /* set if hash of any block failed */ - bool use_tasklet:1; /* try to verify in tasklet before work-queue */ + bool use_bh_wq:1; /* try to verify in BH wq before normal work-queue */ unsigned int digest_size; /* digest size for the current hash algorithm */ unsigned int ahash_reqsize;/* the size of temporary space for crypto */ enum verity_mode mode; /* mode for handling verification errors */ @@ -84,9 +84,10 @@ struct dm_verity_io { sector_t block; unsigned int n_blocks; - bool in_tasklet; + bool in_bh; struct work_struct work; + struct work_struct bh_work; char *recheck_buffer; diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c index b463c28c39ad..7ce8847b3404 100644 --- a/drivers/md/dm-writecache.c +++ b/drivers/md/dm-writecache.c @@ -531,7 +531,7 @@ static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios) req.notify.context = &endio; /* writing via async dm-io (implied by notify.fn above) won't return an error */ - (void) dm_io(&req, 1, ®ion, NULL); + (void) dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); i = j; } @@ -568,7 +568,7 @@ static void ssd_commit_superblock(struct dm_writecache *wc) req.notify.fn = NULL; req.notify.context = NULL; - r = dm_io(&req, 1, ®ion, NULL); + r = dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); if (unlikely(r)) writecache_error(wc, r, "error writing superblock"); } @@ -596,7 +596,7 @@ static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev) req.client = wc->dm_io; req.notify.fn = NULL; - r = dm_io(&req, 1, ®ion, NULL); + r = dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); if (unlikely(r)) writecache_error(wc, r, "error flushing metadata: %d", r); } @@ -990,7 +990,7 @@ static int writecache_read_metadata(struct dm_writecache *wc, sector_t n_sectors req.client = wc->dm_io; req.notify.fn = NULL; - return dm_io(&req, 1, ®ion, NULL); + return dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); } static void writecache_resume(struct dm_target *ti) @@ -2776,5 +2776,5 @@ static struct target_type writecache_target = { module_dm(writecache); MODULE_DESCRIPTION(DM_NAME " writecache target"); -MODULE_AUTHOR("Mikulas Patocka "); +MODULE_AUTHOR("Mikulas Patocka "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 447e132d09b5..06263b0a7b58 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -2948,6 +2948,9 @@ static void __dm_internal_suspend(struct mapped_device *md, unsigned int suspend static void __dm_internal_resume(struct mapped_device *md) { + int r; + struct dm_table *map; + BUG_ON(!md->internal_suspend_count); if (--md->internal_suspend_count) @@ -2956,12 +2959,23 @@ static void __dm_internal_resume(struct mapped_device *md) if (dm_suspended_md(md)) goto done; /* resume from nested suspend */ - /* - * NOTE: existing callers don't need to call dm_table_resume_targets - * (which may fail -- so best to avoid it for now by passing NULL map) - */ - (void) __dm_resume(md, NULL); - + map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); + r = __dm_resume(md, map); + if (r) { + /* + * If a preresume method of some target failed, we are in a + * tricky situation. We can't return an error to the caller. We + * can't fake success because then the "resume" and + * "postsuspend" methods would not be paired correctly, and it + * would break various targets, for example it would cause list + * corruption in the "origin" target. + * + * So, we fake normal suspend here, to make sure that the + * "resume" and "postsuspend" methods will be paired correctly. + */ + DMERR("Preresume method failed: %d", r); + set_bit(DMF_SUSPENDED, &md->flags); + } done: clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); smp_mb__after_atomic(); @@ -3515,5 +3529,5 @@ module_param(swap_bios, int, 0644); MODULE_PARM_DESC(swap_bios, "Maximum allowed inflight swap IOs"); MODULE_DESCRIPTION(DM_NAME " driver"); -MODULE_AUTHOR("Joe Thornber "); +MODULE_AUTHOR("Joe Thornber "); MODULE_LICENSE("GPL"); diff --git a/drivers/md/persistent-data/dm-block-manager.c b/drivers/md/persistent-data/dm-block-manager.c index 0e010e1204aa..b17b54df673b 100644 --- a/drivers/md/persistent-data/dm-block-manager.c +++ b/drivers/md/persistent-data/dm-block-manager.c @@ -656,7 +656,7 @@ EXPORT_SYMBOL_GPL(dm_bm_checksum); /*----------------------------------------------------------------*/ MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Joe Thornber "); +MODULE_AUTHOR("Joe Thornber "); MODULE_DESCRIPTION("Immutable metadata library for dm"); /*----------------------------------------------------------------*/ diff --git a/drivers/media/i2c/ccs/ccs-core.c b/drivers/media/i2c/ccs/ccs-core.c index e21287d50c15..e1ae0f9fad43 100644 --- a/drivers/media/i2c/ccs/ccs-core.c +++ b/drivers/media/i2c/ccs/ccs-core.c @@ -674,7 +674,7 @@ static int ccs_set_ctrl(struct v4l2_ctrl *ctrl) break; } - pm_status = pm_runtime_get_if_active(&client->dev, true); + pm_status = pm_runtime_get_if_active(&client->dev); if (!pm_status) return 0; diff --git a/drivers/media/i2c/ov64a40.c b/drivers/media/i2c/ov64a40.c index 4fba4c2cb064..541bf74581d2 100644 --- a/drivers/media/i2c/ov64a40.c +++ b/drivers/media/i2c/ov64a40.c @@ -3287,7 +3287,7 @@ static int ov64a40_set_ctrl(struct v4l2_ctrl *ctrl) exp_max, 1, exp_val); } - pm_status = pm_runtime_get_if_active(ov64a40->dev, true); + pm_status = pm_runtime_get_if_active(ov64a40->dev); if (!pm_status) return 0; diff --git a/drivers/media/i2c/thp7312.c b/drivers/media/i2c/thp7312.c index 2806887514dc..19bd923a7315 100644 --- a/drivers/media/i2c/thp7312.c +++ b/drivers/media/i2c/thp7312.c @@ -1052,7 +1052,7 @@ static int thp7312_s_ctrl(struct v4l2_ctrl *ctrl) if (ctrl->flags & V4L2_CTRL_FLAG_INACTIVE) return -EINVAL; - if (!pm_runtime_get_if_active(thp7312->dev, true)) + if (!pm_runtime_get_if_active(thp7312->dev)) return 0; switch (ctrl->id) { diff --git a/drivers/media/pci/mgb4/mgb4_core.c b/drivers/media/pci/mgb4/mgb4_core.c index 5bfb8a06202e..9bcf10a77fd3 100644 --- a/drivers/media/pci/mgb4/mgb4_core.c +++ b/drivers/media/pci/mgb4/mgb4_core.c @@ -144,7 +144,7 @@ static int match_spi_adap(struct device *dev, void *data) return to_spi_device(dev) ? 1 : 0; } -static struct spi_master *get_spi_adap(struct platform_device *pdev) +static struct spi_controller *get_spi_adap(struct platform_device *pdev) { struct device *dev; @@ -152,7 +152,7 @@ static struct spi_master *get_spi_adap(struct platform_device *pdev) dev = device_find_child(&pdev->dev, NULL, match_spi_adap); mutex_unlock(&pdev->dev.mutex); - return dev ? container_of(dev, struct spi_master, dev) : NULL; + return dev ? container_of(dev, struct spi_controller, dev) : NULL; } static int init_spi(struct mgb4_dev *mgbdev, u32 devid) @@ -179,7 +179,7 @@ static int init_spi(struct mgb4_dev *mgbdev, u32 devid) }; struct pci_dev *pdev = mgbdev->pdev; struct device *dev = &pdev->dev; - struct spi_master *master; + struct spi_controller *ctlr; struct spi_device *spi_dev; u32 irq; int rv, id; @@ -207,8 +207,8 @@ static int init_spi(struct mgb4_dev *mgbdev, u32 devid) return PTR_ERR(mgbdev->spi_pdev); } - master = get_spi_adap(mgbdev->spi_pdev); - if (!master) { + ctlr = get_spi_adap(mgbdev->spi_pdev); + if (!ctlr) { dev_err(dev, "failed to get SPI adapter\n"); rv = -EINVAL; goto err_pdev; @@ -242,8 +242,8 @@ static int init_spi(struct mgb4_dev *mgbdev, u32 devid) spi_info.platform_data = &mgbdev->flash_data; - spi_dev = spi_new_device(master, &spi_info); - put_device(&master->dev); + spi_dev = spi_new_device(ctlr, &spi_info); + put_device(&ctlr->dev); if (!spi_dev) { dev_err(dev, "failed to create MTD device\n"); rv = -EINVAL; diff --git a/drivers/media/pci/netup_unidvb/netup_unidvb_spi.c b/drivers/media/pci/netup_unidvb/netup_unidvb_spi.c index 526042d8afae..e90aa1c1584c 100644 --- a/drivers/media/pci/netup_unidvb/netup_unidvb_spi.c +++ b/drivers/media/pci/netup_unidvb/netup_unidvb_spi.c @@ -35,7 +35,7 @@ struct netup_spi_regs { struct netup_spi { struct device *dev; - struct spi_master *master; + struct spi_controller *ctlr; struct netup_spi_regs __iomem *regs; u8 __iomem *mmio; spinlock_t lock; @@ -78,7 +78,7 @@ irqreturn_t netup_spi_interrupt(struct netup_spi *spi) reg = readw(&spi->regs->control_stat); if (!(reg & NETUP_SPI_CTRL_IRQ)) { spin_unlock_irqrestore(&spi->lock, flags); - dev_dbg(&spi->master->dev, + dev_dbg(&spi->ctlr->dev, "%s(): not mine interrupt\n", __func__); return IRQ_NONE; } @@ -88,15 +88,15 @@ irqreturn_t netup_spi_interrupt(struct netup_spi *spi) spi->state = SPI_STATE_DONE; wake_up(&spi->waitq); spin_unlock_irqrestore(&spi->lock, flags); - dev_dbg(&spi->master->dev, + dev_dbg(&spi->ctlr->dev, "%s(): SPI interrupt handled\n", __func__); return IRQ_HANDLED; } -static int netup_spi_transfer(struct spi_master *master, +static int netup_spi_transfer(struct spi_controller *ctlr, struct spi_message *msg) { - struct netup_spi *spi = spi_master_get_devdata(master); + struct netup_spi *spi = spi_controller_get_devdata(ctlr); struct spi_transfer *t; int result = 0; u32 tr_size; @@ -131,7 +131,7 @@ static int netup_spi_transfer(struct spi_master *master, NETUP_SPI_CTRL_START | (frag_last ? NETUP_SPI_CTRL_LAST_CS : 0), &spi->regs->control_stat); - dev_dbg(&spi->master->dev, + dev_dbg(&spi->ctlr->dev, "%s(): control_stat 0x%04x\n", __func__, readw(&spi->regs->control_stat)); wait_event_timeout(spi->waitq, @@ -144,11 +144,11 @@ static int netup_spi_transfer(struct spi_master *master, } } else { if (spi->state == SPI_STATE_START) { - dev_dbg(&spi->master->dev, + dev_dbg(&spi->ctlr->dev, "%s(): transfer timeout\n", __func__); } else { - dev_dbg(&spi->master->dev, + dev_dbg(&spi->ctlr->dev, "%s(): invalid state %d\n", __func__, spi->state); } @@ -161,7 +161,7 @@ static int netup_spi_transfer(struct spi_master *master, } done: msg->status = result; - spi_finalize_current_message(master); + spi_finalize_current_message(ctlr); return result; } @@ -172,30 +172,30 @@ static int netup_spi_setup(struct spi_device *spi) int netup_spi_init(struct netup_unidvb_dev *ndev) { - struct spi_master *master; + struct spi_controller *ctlr; struct netup_spi *nspi; - master = devm_spi_alloc_master(&ndev->pci_dev->dev, - sizeof(struct netup_spi)); - if (!master) { + ctlr = devm_spi_alloc_master(&ndev->pci_dev->dev, + sizeof(struct netup_spi)); + if (!ctlr) { dev_err(&ndev->pci_dev->dev, "%s(): unable to alloc SPI master\n", __func__); return -EINVAL; } - nspi = spi_master_get_devdata(master); - master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_LSB_FIRST; - master->bus_num = -1; - master->num_chipselect = 1; - master->transfer_one_message = netup_spi_transfer; - master->setup = netup_spi_setup; + nspi = spi_controller_get_devdata(ctlr); + ctlr->mode_bits = SPI_CPOL | SPI_CPHA | SPI_LSB_FIRST; + ctlr->bus_num = -1; + ctlr->num_chipselect = 1; + ctlr->transfer_one_message = netup_spi_transfer; + ctlr->setup = netup_spi_setup; spin_lock_init(&nspi->lock); init_waitqueue_head(&nspi->waitq); - nspi->master = master; + nspi->ctlr = ctlr; nspi->regs = (struct netup_spi_regs __iomem *)(ndev->bmmio0 + 0x4000); writew(2, &nspi->regs->clock_divider); writew(NETUP_UNIDVB_IRQ_SPI, ndev->bmmio0 + REG_IMASK_SET); ndev->spi = nspi; - if (spi_register_master(master)) { + if (spi_register_controller(ctlr)) { ndev->spi = NULL; dev_err(&ndev->pci_dev->dev, "%s(): unable to register SPI bus\n", __func__); @@ -207,8 +207,8 @@ int netup_spi_init(struct netup_unidvb_dev *ndev) ndev->pci_bus, ndev->pci_slot, ndev->pci_func); - if (!spi_new_device(master, &netup_spi_board)) { - spi_unregister_master(master); + if (!spi_new_device(ctlr, &netup_spi_board)) { + spi_unregister_controller(ctlr); ndev->spi = NULL; dev_err(&ndev->pci_dev->dev, "%s(): unable to create SPI device\n", __func__); @@ -227,7 +227,7 @@ void netup_spi_release(struct netup_unidvb_dev *ndev) if (!spi) return; - spi_unregister_master(spi->master); + spi_unregister_controller(spi->ctlr); spin_lock_irqsave(&spi->lock, flags); reg = readw(&spi->regs->control_stat); writew(reg | NETUP_SPI_CTRL_IRQ, &spi->regs->control_stat); diff --git a/drivers/media/platform/qcom/venus/core.c b/drivers/media/platform/qcom/venus/core.c index a712dd4f02a5..ce206b709754 100644 --- a/drivers/media/platform/qcom/venus/core.c +++ b/drivers/media/platform/qcom/venus/core.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -114,7 +115,8 @@ static void venus_sys_error_handler(struct work_struct *work) pm_runtime_put_sync(core->dev); for (i = 0; i < max_attempts; i++) { - if (!core->pmdomains[0] || !pm_runtime_active(core->pmdomains[0])) + if (!core->pmdomains || + !pm_runtime_active(core->pmdomains->pd_devs[0])) break; usleep_range(1000, 1500); } @@ -705,7 +707,7 @@ static const struct venus_resources sdm845_res_v2 = { .vcodec0_clks = { "vcodec0_core", "vcodec0_bus" }, .vcodec1_clks = { "vcodec1_core", "vcodec1_bus" }, .vcodec_clks_num = 2, - .vcodec_pmdomains = { "venus", "vcodec0", "vcodec1" }, + .vcodec_pmdomains = (const char *[]) { "venus", "vcodec0", "vcodec1" }, .vcodec_pmdomains_num = 3, .opp_pmdomain = (const char *[]) { "cx", NULL }, .vcodec_num = 2, @@ -754,7 +756,7 @@ static const struct venus_resources sc7180_res = { .clks_num = 3, .vcodec0_clks = { "vcodec0_core", "vcodec0_bus" }, .vcodec_clks_num = 2, - .vcodec_pmdomains = { "venus", "vcodec0" }, + .vcodec_pmdomains = (const char *[]) { "venus", "vcodec0" }, .vcodec_pmdomains_num = 2, .opp_pmdomain = (const char *[]) { "cx", NULL }, .vcodec_num = 1, @@ -811,7 +813,7 @@ static const struct venus_resources sm8250_res = { .resets_num = 2, .vcodec0_clks = { "vcodec0_core" }, .vcodec_clks_num = 1, - .vcodec_pmdomains = { "venus", "vcodec0" }, + .vcodec_pmdomains = (const char *[]) { "venus", "vcodec0" }, .vcodec_pmdomains_num = 2, .opp_pmdomain = (const char *[]) { "mx", NULL }, .vcodec_num = 1, @@ -870,7 +872,7 @@ static const struct venus_resources sc7280_res = { .clks_num = 3, .vcodec0_clks = {"vcodec_core", "vcodec_bus"}, .vcodec_clks_num = 2, - .vcodec_pmdomains = { "venus", "vcodec0" }, + .vcodec_pmdomains = (const char *[]) { "venus", "vcodec0" }, .vcodec_pmdomains_num = 2, .opp_pmdomain = (const char *[]) { "cx", NULL }, .vcodec_num = 1, diff --git a/drivers/media/platform/qcom/venus/core.h b/drivers/media/platform/qcom/venus/core.h index 4a633261ece4..7ef341bf21cc 100644 --- a/drivers/media/platform/qcom/venus/core.h +++ b/drivers/media/platform/qcom/venus/core.h @@ -25,7 +25,6 @@ #define VIDC_CLKS_NUM_MAX 4 #define VIDC_VCODEC_CLKS_NUM_MAX 2 -#define VIDC_PMDOMAINS_NUM_MAX 3 #define VIDC_RESETS_NUM_MAX 2 extern int venus_fw_debug; @@ -72,7 +71,7 @@ struct venus_resources { const char * const vcodec0_clks[VIDC_VCODEC_CLKS_NUM_MAX]; const char * const vcodec1_clks[VIDC_VCODEC_CLKS_NUM_MAX]; unsigned int vcodec_clks_num; - const char * const vcodec_pmdomains[VIDC_PMDOMAINS_NUM_MAX]; + const char **vcodec_pmdomains; unsigned int vcodec_pmdomains_num; const char **opp_pmdomain; unsigned int vcodec_num; @@ -134,7 +133,7 @@ struct venus_format { * @video_path: an interconnect handle to video to/from memory path * @cpucfg_path: an interconnect handle to cpu configuration path * @has_opp_table: does OPP table exist - * @pmdomains: an array of pmdomains struct device pointers + * @pmdomains: a pointer to a list of pmdomains * @opp_dl_venus: an device-link for device OPP * @opp_pmdomain: an OPP power-domain * @resets: an array of reset signals @@ -187,7 +186,7 @@ struct venus_core { struct icc_path *video_path; struct icc_path *cpucfg_path; bool has_opp_table; - struct device *pmdomains[VIDC_PMDOMAINS_NUM_MAX]; + struct dev_pm_domain_list *pmdomains; struct device_link *opp_dl_venus; struct device *opp_pmdomain; struct reset_control *resets[VIDC_RESETS_NUM_MAX]; diff --git a/drivers/media/platform/qcom/venus/pm_helpers.c b/drivers/media/platform/qcom/venus/pm_helpers.c index a1b127caa90a..502822059498 100644 --- a/drivers/media/platform/qcom/venus/pm_helpers.c +++ b/drivers/media/platform/qcom/venus/pm_helpers.c @@ -455,7 +455,7 @@ static int poweroff_coreid(struct venus_core *core, unsigned int coreid_mask) if (ret) return ret; - ret = pm_runtime_put_sync(core->pmdomains[1]); + ret = pm_runtime_put_sync(core->pmdomains->pd_devs[1]); if (ret < 0) return ret; } @@ -471,7 +471,7 @@ static int poweroff_coreid(struct venus_core *core, unsigned int coreid_mask) if (ret) return ret; - ret = pm_runtime_put_sync(core->pmdomains[2]); + ret = pm_runtime_put_sync(core->pmdomains->pd_devs[2]); if (ret < 0) return ret; } @@ -484,7 +484,7 @@ static int poweron_coreid(struct venus_core *core, unsigned int coreid_mask) int ret; if (coreid_mask & VIDC_CORE_ID_1) { - ret = pm_runtime_get_sync(core->pmdomains[1]); + ret = pm_runtime_get_sync(core->pmdomains->pd_devs[1]); if (ret < 0) return ret; @@ -502,7 +502,7 @@ static int poweron_coreid(struct venus_core *core, unsigned int coreid_mask) } if (coreid_mask & VIDC_CORE_ID_2) { - ret = pm_runtime_get_sync(core->pmdomains[2]); + ret = pm_runtime_get_sync(core->pmdomains->pd_devs[2]); if (ret < 0) return ret; @@ -860,19 +860,18 @@ static int vcodec_domains_get(struct venus_core *core) struct device **opp_virt_dev; struct device *dev = core->dev; const struct venus_resources *res = core->res; - struct device *pd; - unsigned int i; + struct dev_pm_domain_attach_data vcodec_data = { + .pd_names = res->vcodec_pmdomains, + .num_pd_names = res->vcodec_pmdomains_num, + .pd_flags = PD_FLAG_NO_DEV_LINK, + }; if (!res->vcodec_pmdomains_num) goto skip_pmdomains; - for (i = 0; i < res->vcodec_pmdomains_num; i++) { - pd = dev_pm_domain_attach_by_name(dev, - res->vcodec_pmdomains[i]); - if (IS_ERR_OR_NULL(pd)) - return pd ? PTR_ERR(pd) : -ENODATA; - core->pmdomains[i] = pd; - } + ret = dev_pm_domain_attach_list(dev, &vcodec_data, &core->pmdomains); + if (ret < 0) + return ret; skip_pmdomains: if (!core->res->opp_pmdomain) @@ -896,30 +895,14 @@ skip_pmdomains: return 0; opp_attach_err: - for (i = 0; i < res->vcodec_pmdomains_num; i++) { - if (IS_ERR_OR_NULL(core->pmdomains[i])) - continue; - dev_pm_domain_detach(core->pmdomains[i], true); - } - + dev_pm_domain_detach_list(core->pmdomains); return ret; } static void vcodec_domains_put(struct venus_core *core) { - const struct venus_resources *res = core->res; - unsigned int i; + dev_pm_domain_detach_list(core->pmdomains); - if (!res->vcodec_pmdomains_num) - goto skip_pmdomains; - - for (i = 0; i < res->vcodec_pmdomains_num; i++) { - if (IS_ERR_OR_NULL(core->pmdomains[i])) - continue; - dev_pm_domain_detach(core->pmdomains[i], true); - } - -skip_pmdomains: if (!core->has_opp_table) return; @@ -1035,7 +1018,8 @@ static void core_put_v4(struct venus_core *core) static int core_power_v4(struct venus_core *core, int on) { struct device *dev = core->dev; - struct device *pmctrl = core->pmdomains[0]; + struct device *pmctrl = core->pmdomains ? + core->pmdomains->pd_devs[0] : NULL; int ret = 0; if (on == POWER_ON) { diff --git a/drivers/media/usb/msi2500/msi2500.c b/drivers/media/usb/msi2500/msi2500.c index 9759996ee6a4..5138486abfa0 100644 --- a/drivers/media/usb/msi2500/msi2500.c +++ b/drivers/media/usb/msi2500/msi2500.c @@ -107,7 +107,7 @@ struct msi2500_dev { struct video_device vdev; struct v4l2_device v4l2_dev; struct v4l2_subdev *v4l2_subdev; - struct spi_master *master; + struct spi_controller *ctlr; /* videobuf2 queue and queued buffers list */ struct vb2_queue vb_queue; @@ -574,7 +574,7 @@ static void msi2500_disconnect(struct usb_interface *intf) dev->udev = NULL; v4l2_device_disconnect(&dev->v4l2_dev); video_unregister_device(&dev->vdev); - spi_unregister_master(dev->master); + spi_unregister_controller(dev->ctlr); mutex_unlock(&dev->v4l2_lock); mutex_unlock(&dev->vb_queue_lock); @@ -1136,10 +1136,10 @@ static void msi2500_video_release(struct v4l2_device *v) kfree(dev); } -static int msi2500_transfer_one_message(struct spi_master *master, +static int msi2500_transfer_one_message(struct spi_controller *ctlr, struct spi_message *m) { - struct msi2500_dev *dev = spi_master_get_devdata(master); + struct msi2500_dev *dev = spi_controller_get_devdata(ctlr); struct spi_transfer *t; int ret = 0; u32 data; @@ -1154,7 +1154,7 @@ static int msi2500_transfer_one_message(struct spi_master *master, } m->status = ret; - spi_finalize_current_message(master); + spi_finalize_current_message(ctlr); return ret; } @@ -1163,7 +1163,7 @@ static int msi2500_probe(struct usb_interface *intf, { struct msi2500_dev *dev; struct v4l2_subdev *sd; - struct spi_master *master; + struct spi_controller *ctlr; int ret; static struct spi_board_info board_info = { .modalias = "msi001", @@ -1220,30 +1220,30 @@ static int msi2500_probe(struct usb_interface *intf, } /* SPI master adapter */ - master = spi_alloc_master(dev->dev, 0); - if (master == NULL) { + ctlr = spi_alloc_master(dev->dev, 0); + if (ctlr == NULL) { ret = -ENOMEM; goto err_unregister_v4l2_dev; } - dev->master = master; - master->bus_num = -1; - master->num_chipselect = 1; - master->transfer_one_message = msi2500_transfer_one_message; - spi_master_set_devdata(master, dev); - ret = spi_register_master(master); + dev->ctlr = ctlr; + ctlr->bus_num = -1; + ctlr->num_chipselect = 1; + ctlr->transfer_one_message = msi2500_transfer_one_message; + spi_controller_set_devdata(ctlr, dev); + ret = spi_register_controller(ctlr); if (ret) { - spi_master_put(master); + spi_controller_put(ctlr); goto err_unregister_v4l2_dev; } /* load v4l2 subdevice */ - sd = v4l2_spi_new_subdev(&dev->v4l2_dev, master, &board_info); + sd = v4l2_spi_new_subdev(&dev->v4l2_dev, ctlr, &board_info); dev->v4l2_subdev = sd; if (sd == NULL) { dev_err(dev->dev, "cannot get v4l2 subdevice\n"); ret = -ENODEV; - goto err_unregister_master; + goto err_unregister_controller; } /* Register controls */ @@ -1276,8 +1276,8 @@ static int msi2500_probe(struct usb_interface *intf, return 0; err_free_controls: v4l2_ctrl_handler_free(&dev->hdl); -err_unregister_master: - spi_unregister_master(dev->master); +err_unregister_controller: + spi_unregister_controller(dev->ctlr); err_unregister_v4l2_dev: v4l2_device_unregister(&dev->v4l2_dev); err_free_mem: diff --git a/drivers/media/v4l2-core/v4l2-spi.c b/drivers/media/v4l2-core/v4l2-spi.c index eadecdff7349..a7092c3930d6 100644 --- a/drivers/media/v4l2-core/v4l2-spi.c +++ b/drivers/media/v4l2-core/v4l2-spi.c @@ -34,7 +34,7 @@ void v4l2_spi_subdev_init(struct v4l2_subdev *sd, struct spi_device *spi, EXPORT_SYMBOL_GPL(v4l2_spi_subdev_init); struct v4l2_subdev *v4l2_spi_new_subdev(struct v4l2_device *v4l2_dev, - struct spi_master *master, + struct spi_controller *ctlr, struct spi_board_info *info) { struct v4l2_subdev *sd = NULL; @@ -45,7 +45,7 @@ struct v4l2_subdev *v4l2_spi_new_subdev(struct v4l2_device *v4l2_dev, if (info->modalias[0]) request_module(info->modalias); - spi = spi_new_device(master, info); + spi = spi_new_device(ctlr, info); if (!spi || !spi->dev.driver) goto error; diff --git a/drivers/memstick/core/memstick.c b/drivers/memstick/core/memstick.c index bbfaf6536903..23fea51ecbdd 100644 --- a/drivers/memstick/core/memstick.c +++ b/drivers/memstick/core/memstick.c @@ -164,7 +164,7 @@ static struct attribute *memstick_dev_attrs[] = { }; ATTRIBUTE_GROUPS(memstick_dev); -static struct bus_type memstick_bus_type = { +static const struct bus_type memstick_bus_type = { .name = "memstick", .dev_groups = memstick_dev_groups, .match = memstick_bus_match, diff --git a/drivers/misc/gehc-achc.c b/drivers/misc/gehc-achc.c index 4c9c5394da6f..b8fca4d393c6 100644 --- a/drivers/misc/gehc-achc.c +++ b/drivers/misc/gehc-achc.c @@ -65,7 +65,7 @@ static int ezport_start_programming(struct spi_device *spi, struct gpio_desc *re struct spi_transfer release_cs = { }; int ret; - spi_bus_lock(spi->master); + spi_bus_lock(spi->controller); /* assert chip select */ spi_message_init(&msg); @@ -85,16 +85,16 @@ static int ezport_start_programming(struct spi_device *spi, struct gpio_desc *re ret = spi_sync_locked(spi, &msg); fail: - spi_bus_unlock(spi->master); + spi_bus_unlock(spi->controller); return ret; } static void ezport_stop_programming(struct spi_device *spi, struct gpio_desc *reset) { /* reset without asserted chip select to return into normal mode */ - spi_bus_lock(spi->master); + spi_bus_lock(spi->controller); ezport_reset(reset); - spi_bus_unlock(spi->master); + spi_bus_unlock(spi->controller); } static int ezport_get_status_register(struct spi_device *spi) diff --git a/drivers/misc/lkdtm/heap.c b/drivers/misc/lkdtm/heap.c index 4f467d3972a6..b1b316f99703 100644 --- a/drivers/misc/lkdtm/heap.c +++ b/drivers/misc/lkdtm/heap.c @@ -48,7 +48,7 @@ static void lkdtm_VMALLOC_LINEAR_OVERFLOW(void) * correctly. * * This should get caught by either memory tagging, KASan, or by using - * CONFIG_SLUB_DEBUG=y and slub_debug=ZF (or CONFIG_SLUB_DEBUG_ON=y). + * CONFIG_SLUB_DEBUG=y and slab_debug=ZF (or CONFIG_SLUB_DEBUG_ON=y). */ static void lkdtm_SLAB_LINEAR_OVERFLOW(void) { diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index 32d49100dff5..64a3492e8002 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -144,7 +144,7 @@ struct mmc_blk_data { static dev_t mmc_rpmb_devt; /* Bus type for RPMB character devices */ -static struct bus_type mmc_rpmb_bus_type = { +static const struct bus_type mmc_rpmb_bus_type = { .name = "mmc_rpmb", }; @@ -206,7 +206,7 @@ static void mmc_blk_kref_release(struct kref *ref) int devidx; devidx = mmc_get_devidx(md->disk); - ida_simple_remove(&mmc_blk_ida, devidx); + ida_free(&mmc_blk_ida, devidx); mutex_lock(&open_lock); md->disk->private_data = NULL; @@ -874,10 +874,11 @@ static const struct block_device_operations mmc_bdops = { static int mmc_blk_part_switch_pre(struct mmc_card *card, unsigned int part_type) { - const unsigned int mask = EXT_CSD_PART_CONFIG_ACC_RPMB; + const unsigned int mask = EXT_CSD_PART_CONFIG_ACC_MASK; + const unsigned int rpmb = EXT_CSD_PART_CONFIG_ACC_RPMB; int ret = 0; - if ((part_type & mask) == mask) { + if ((part_type & mask) == rpmb) { if (card->ext_csd.cmdq_en) { ret = mmc_cmdq_disable(card); if (ret) @@ -892,10 +893,11 @@ static int mmc_blk_part_switch_pre(struct mmc_card *card, static int mmc_blk_part_switch_post(struct mmc_card *card, unsigned int part_type) { - const unsigned int mask = EXT_CSD_PART_CONFIG_ACC_RPMB; + const unsigned int mask = EXT_CSD_PART_CONFIG_ACC_MASK; + const unsigned int rpmb = EXT_CSD_PART_CONFIG_ACC_RPMB; int ret = 0; - if ((part_type & mask) == mask) { + if ((part_type & mask) == rpmb) { mmc_retune_unpause(card->host); if (card->reenable_cmdq && !card->ext_csd.cmdq_en) ret = mmc_cmdq_enable(card); @@ -2467,7 +2469,7 @@ static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card, bool cache_enabled = false; bool fua_enabled = false; - devidx = ida_simple_get(&mmc_blk_ida, 0, max_devices, GFP_KERNEL); + devidx = ida_alloc_max(&mmc_blk_ida, max_devices - 1, GFP_KERNEL); if (devidx < 0) { /* * We get -ENOSPC because there are no more any available @@ -2577,7 +2579,7 @@ static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card, err_kfree: kfree(md); out: - ida_simple_remove(&mmc_blk_ida, devidx); + ida_free(&mmc_blk_ida, devidx); return ERR_PTR(ret); } @@ -2703,7 +2705,7 @@ static void mmc_blk_rpmb_device_release(struct device *dev) { struct mmc_rpmb_data *rpmb = dev_get_drvdata(dev); - ida_simple_remove(&mmc_rpmb_ida, rpmb->id); + ida_free(&mmc_rpmb_ida, rpmb->id); kfree(rpmb); } @@ -2719,13 +2721,13 @@ static int mmc_blk_alloc_rpmb_part(struct mmc_card *card, struct mmc_rpmb_data *rpmb; /* This creates the minor number for the RPMB char device */ - devidx = ida_simple_get(&mmc_rpmb_ida, 0, max_devices, GFP_KERNEL); + devidx = ida_alloc_max(&mmc_rpmb_ida, max_devices - 1, GFP_KERNEL); if (devidx < 0) return devidx; rpmb = kzalloc(sizeof(*rpmb), GFP_KERNEL); if (!rpmb) { - ida_simple_remove(&mmc_rpmb_ida, devidx); + ida_free(&mmc_rpmb_ida, devidx); return -ENOMEM; } diff --git a/drivers/mmc/core/bus.c b/drivers/mmc/core/bus.c index 0af96548e7da..0ddaee0eae54 100644 --- a/drivers/mmc/core/bus.c +++ b/drivers/mmc/core/bus.c @@ -214,7 +214,7 @@ static const struct dev_pm_ops mmc_bus_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS(mmc_bus_suspend, mmc_bus_resume) }; -static struct bus_type mmc_bus_type = { +static const struct bus_type mmc_bus_type = { .name = "mmc", .dev_groups = mmc_dev_groups, .uevent = mmc_bus_uevent, @@ -272,7 +272,7 @@ static void mmc_release_card(struct device *dev) /* * Allocate and initialise a new MMC card structure. */ -struct mmc_card *mmc_alloc_card(struct mmc_host *host, struct device_type *type) +struct mmc_card *mmc_alloc_card(struct mmc_host *host, const struct device_type *type) { struct mmc_card *card; diff --git a/drivers/mmc/core/bus.h b/drivers/mmc/core/bus.h index 3996b191b68d..cfd0d02d3420 100644 --- a/drivers/mmc/core/bus.h +++ b/drivers/mmc/core/bus.h @@ -23,7 +23,7 @@ static ssize_t mmc_##name##_show (struct device *dev, struct device_attribute *a static DEVICE_ATTR(name, S_IRUGO, mmc_##name##_show, NULL) struct mmc_card *mmc_alloc_card(struct mmc_host *host, - struct device_type *type); + const struct device_type *type); int mmc_add_card(struct mmc_card *card); void mmc_remove_card(struct mmc_card *card); diff --git a/drivers/mmc/core/host.c b/drivers/mmc/core/host.c index cf396e8f34e9..8f8781d6c25e 100644 --- a/drivers/mmc/core/host.c +++ b/drivers/mmc/core/host.c @@ -76,7 +76,7 @@ static void mmc_host_classdev_release(struct device *dev) struct mmc_host *host = cls_dev_to_mmc_host(dev); wakeup_source_unregister(host->ws); if (of_alias_get_id(host->parent->of_node, "mmc") < 0) - ida_simple_remove(&mmc_host_ida, host->index); + ida_free(&mmc_host_ida, host->index); kfree(host); } @@ -88,7 +88,7 @@ static int mmc_host_classdev_shutdown(struct device *dev) return 0; } -static struct class mmc_host_class = { +static const struct class mmc_host_class = { .name = "mmc_host", .dev_release = mmc_host_classdev_release, .shutdown_pre = mmc_host_classdev_shutdown, @@ -234,10 +234,8 @@ static void mmc_of_parse_timing_phase(struct device *dev, const char *prop, } void -mmc_of_parse_clk_phase(struct mmc_host *host, struct mmc_clk_phase_map *map) +mmc_of_parse_clk_phase(struct device *dev, struct mmc_clk_phase_map *map) { - struct device *dev = host->parent; - mmc_of_parse_timing_phase(dev, "clk-phase-legacy", &map->phase[MMC_TIMING_LEGACY]); mmc_of_parse_timing_phase(dev, "clk-phase-mmc-hs", @@ -538,7 +536,8 @@ struct mmc_host *mmc_alloc_host(int extra, struct device *dev) min_idx = mmc_first_nonreserved_index(); max_idx = 0; - index = ida_simple_get(&mmc_host_ida, min_idx, max_idx, GFP_KERNEL); + index = ida_alloc_range(&mmc_host_ida, min_idx, max_idx - 1, + GFP_KERNEL); if (index < 0) { kfree(host); return NULL; diff --git a/drivers/mmc/core/mmc.c b/drivers/mmc/core/mmc.c index 58ed7193a3ca..5b2f7c285461 100644 --- a/drivers/mmc/core/mmc.c +++ b/drivers/mmc/core/mmc.c @@ -883,7 +883,7 @@ static struct attribute *mmc_std_attrs[] = { }; ATTRIBUTE_GROUPS(mmc_std); -static struct device_type mmc_type = { +static const struct device_type mmc_type = { .groups = mmc_std_groups, }; diff --git a/drivers/mmc/core/queue.c b/drivers/mmc/core/queue.c index 2ae60d208cdf..241cdc2b2a2a 100644 --- a/drivers/mmc/core/queue.c +++ b/drivers/mmc/core/queue.c @@ -353,9 +353,6 @@ static struct gendisk *mmc_alloc_disk(struct mmc_queue *mq, if (mmc_can_erase(card)) mmc_queue_setup_discard(card, &lim); - if (!mmc_dev(host)->dma_mask || !*mmc_dev(host)->dma_mask) - lim.bounce = BLK_BOUNCE_HIGH; - lim.max_hw_sectors = min(host->max_blk_count, host->max_req_size / 512); if (mmc_card_mmc(card) && card->ext_csd.data_sector_size) diff --git a/drivers/mmc/core/sd.c b/drivers/mmc/core/sd.c index c3e554344c99..1c8148cdda50 100644 --- a/drivers/mmc/core/sd.c +++ b/drivers/mmc/core/sd.c @@ -805,7 +805,7 @@ static const struct attribute_group sd_std_group = { }; __ATTRIBUTE_GROUPS(sd_std); -struct device_type sd_type = { +const struct device_type sd_type = { .groups = sd_std_groups, }; diff --git a/drivers/mmc/core/sd.h b/drivers/mmc/core/sd.h index 1af5a038bae9..fe6dd46927a4 100644 --- a/drivers/mmc/core/sd.h +++ b/drivers/mmc/core/sd.h @@ -4,7 +4,7 @@ #include -extern struct device_type sd_type; +extern const struct device_type sd_type; struct mmc_host; struct mmc_card; diff --git a/drivers/mmc/core/sdio.c b/drivers/mmc/core/sdio.c index 5914516df2f7..4fb247fde5c0 100644 --- a/drivers/mmc/core/sdio.c +++ b/drivers/mmc/core/sdio.c @@ -66,7 +66,7 @@ static struct attribute *sdio_std_attrs[] = { }; ATTRIBUTE_GROUPS(sdio_std); -static struct device_type sdio_type = { +static const struct device_type sdio_type = { .groups = sdio_std_groups, }; diff --git a/drivers/mmc/core/sdio_bus.c b/drivers/mmc/core/sdio_bus.c index 47a48e902a24..71d885fbc228 100644 --- a/drivers/mmc/core/sdio_bus.c +++ b/drivers/mmc/core/sdio_bus.c @@ -244,7 +244,7 @@ static const struct dev_pm_ops sdio_bus_pm_ops = { ) }; -static struct bus_type sdio_bus_type = { +static const struct bus_type sdio_bus_type = { .name = "sdio", .dev_groups = sdio_dev_groups, .match = sdio_bus_match, diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig index 81f2c4e05287..aebc587f77a7 100644 --- a/drivers/mmc/host/Kconfig +++ b/drivers/mmc/host/Kconfig @@ -798,6 +798,15 @@ config MMC_DW_HI3798CV200 Synopsys DesignWare Memory Card Interface driver. Select this option for platforms based on HiSilicon Hi3798CV200 SoC. +config MMC_DW_HI3798MV200 + tristate "Hi3798MV200 specific extensions for Synopsys DW Memory Card Interface" + depends on MMC_DW + select MMC_DW_PLTFM + help + This selects support for HiSilicon Hi3798MV200 SoC specific extensions to the + Synopsys DesignWare Memory Card Interface driver. Select this option + for platforms based on HiSilicon Hi3798MV200 SoC. + config MMC_DW_K3 tristate "K3 specific extensions for Synopsys DW Memory Card Interface" depends on MMC_DW diff --git a/drivers/mmc/host/Makefile b/drivers/mmc/host/Makefile index d0be4465f3ec..f53f86d200ac 100644 --- a/drivers/mmc/host/Makefile +++ b/drivers/mmc/host/Makefile @@ -51,6 +51,7 @@ obj-$(CONFIG_MMC_DW_PLTFM) += dw_mmc-pltfm.o obj-$(CONFIG_MMC_DW_BLUEFIELD) += dw_mmc-bluefield.o obj-$(CONFIG_MMC_DW_EXYNOS) += dw_mmc-exynos.o obj-$(CONFIG_MMC_DW_HI3798CV200) += dw_mmc-hi3798cv200.o +obj-$(CONFIG_MMC_DW_HI3798MV200) += dw_mmc-hi3798mv200.o obj-$(CONFIG_MMC_DW_K3) += dw_mmc-k3.o obj-$(CONFIG_MMC_DW_PCI) += dw_mmc-pci.o obj-$(CONFIG_MMC_DW_ROCKCHIP) += dw_mmc-rockchip.o diff --git a/drivers/mmc/host/davinci_mmc.c b/drivers/mmc/host/davinci_mmc.c index ee3b1a4e0848..8bd938919687 100644 --- a/drivers/mmc/host/davinci_mmc.c +++ b/drivers/mmc/host/davinci_mmc.c @@ -180,12 +180,6 @@ struct mmc_davinci_host { #define DAVINCI_MMC_DATADIR_WRITE 2 unsigned char data_dir; - /* buffer is used during PIO of one scatterlist segment, and - * is updated along with buffer_bytes_left. bytes_left applies - * to all N blocks of the PIO transfer. - */ - u8 *buffer; - u32 buffer_bytes_left; u32 bytes_left; struct dma_chan *dma_tx; @@ -196,8 +190,8 @@ struct mmc_davinci_host { bool active_request; /* For PIO we walk scatterlists one segment at a time. */ + struct sg_mapping_iter sg_miter; unsigned int sg_len; - struct scatterlist *sg; /* Version of the MMC/SD controller */ u8 version; @@ -213,30 +207,22 @@ struct mmc_davinci_host { static irqreturn_t mmc_davinci_irq(int irq, void *dev_id); /* PIO only */ -static void mmc_davinci_sg_to_buf(struct mmc_davinci_host *host) -{ - host->buffer_bytes_left = sg_dma_len(host->sg); - host->buffer = sg_virt(host->sg); - if (host->buffer_bytes_left > host->bytes_left) - host->buffer_bytes_left = host->bytes_left; -} - static void davinci_fifo_data_trans(struct mmc_davinci_host *host, unsigned int n) { + struct sg_mapping_iter *sgm = &host->sg_miter; u8 *p; unsigned int i; - if (host->buffer_bytes_left == 0) { - host->sg = sg_next(host->data->sg); - mmc_davinci_sg_to_buf(host); + /* + * By adjusting sgm->consumed this will give a pointer to the + * current index into the sgm. + */ + if (!sg_miter_next(sgm)) { + dev_err(mmc_dev(host->mmc), "ran out of sglist prematurely\n"); + return; } - - p = host->buffer; - if (n > host->buffer_bytes_left) - n = host->buffer_bytes_left; - host->buffer_bytes_left -= n; - host->bytes_left -= n; + p = sgm->addr; /* NOTE: we never transfer more than rw_threshold bytes * to/from the fifo here; there's no I/O overlap. @@ -261,7 +247,9 @@ static void davinci_fifo_data_trans(struct mmc_davinci_host *host, p = p + (n & 3); } } - host->buffer = p; + + sgm->consumed = n; + host->bytes_left -= n; } static void mmc_davinci_start_command(struct mmc_davinci_host *host, @@ -517,6 +505,7 @@ mmc_davinci_prepare_data(struct mmc_davinci_host *host, struct mmc_request *req) int fifo_lev = (rw_threshold == 32) ? MMCFIFOCTL_FIFOLEV : 0; int timeout; struct mmc_data *data = req->data; + unsigned int flags = SG_MITER_ATOMIC; /* Used from IRQ */ if (host->version == MMC_CTLR_VERSION_2) fifo_lev = (rw_threshold == 64) ? MMCFIFOCTL_FIFOLEV : 0; @@ -545,12 +534,14 @@ mmc_davinci_prepare_data(struct mmc_davinci_host *host, struct mmc_request *req) /* Configure the FIFO */ if (data->flags & MMC_DATA_WRITE) { + flags |= SG_MITER_FROM_SG; host->data_dir = DAVINCI_MMC_DATADIR_WRITE; writel(fifo_lev | MMCFIFOCTL_FIFODIR_WR | MMCFIFOCTL_FIFORST, host->base + DAVINCI_MMCFIFOCTL); writel(fifo_lev | MMCFIFOCTL_FIFODIR_WR, host->base + DAVINCI_MMCFIFOCTL); } else { + flags |= SG_MITER_TO_SG; host->data_dir = DAVINCI_MMC_DATADIR_READ; writel(fifo_lev | MMCFIFOCTL_FIFODIR_RD | MMCFIFOCTL_FIFORST, host->base + DAVINCI_MMCFIFOCTL); @@ -558,7 +549,6 @@ mmc_davinci_prepare_data(struct mmc_davinci_host *host, struct mmc_request *req) host->base + DAVINCI_MMCFIFOCTL); } - host->buffer = NULL; host->bytes_left = data->blocks * data->blksz; /* For now we try to use DMA whenever we won't need partial FIFO @@ -576,8 +566,7 @@ mmc_davinci_prepare_data(struct mmc_davinci_host *host, struct mmc_request *req) } else { /* Revert to CPU Copy */ host->sg_len = data->sg_len; - host->sg = host->data->sg; - mmc_davinci_sg_to_buf(host); + sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags); } } @@ -843,6 +832,8 @@ davinci_abort_data(struct mmc_davinci_host *host, struct mmc_data *data) { mmc_davinci_reset_ctrl(host, 1); mmc_davinci_reset_ctrl(host, 0); + if (!host->do_dma) + sg_miter_stop(&host->sg_miter); } static irqreturn_t mmc_davinci_sdio_irq(int irq, void *dev_id) @@ -919,11 +910,13 @@ static irqreturn_t mmc_davinci_irq(int irq, void *dev_id) if (qstatus & MMCST0_DATDNE) { /* All blocks sent/received, and CRC checks passed */ if (data != NULL) { - if ((host->do_dma == 0) && (host->bytes_left > 0)) { - /* if datasize < rw_threshold - * no RX ints are generated - */ - davinci_fifo_data_trans(host, host->bytes_left); + if (!host->do_dma) { + if (host->bytes_left > 0) + /* if datasize < rw_threshold + * no RX ints are generated + */ + davinci_fifo_data_trans(host, host->bytes_left); + sg_miter_stop(&host->sg_miter); } end_transfer = 1; data->bytes_xfered = data->blocks * data->blksz; diff --git a/drivers/mmc/host/dw_mmc-exynos.c b/drivers/mmc/host/dw_mmc-exynos.c index 698408e8bad0..6dc057718d2c 100644 --- a/drivers/mmc/host/dw_mmc-exynos.c +++ b/drivers/mmc/host/dw_mmc-exynos.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include diff --git a/drivers/mmc/host/dw_mmc-hi3798cv200.c b/drivers/mmc/host/dw_mmc-hi3798cv200.c index e9470c50a348..61923a518369 100644 --- a/drivers/mmc/host/dw_mmc-hi3798cv200.c +++ b/drivers/mmc/host/dw_mmc-hi3798cv200.c @@ -201,4 +201,3 @@ module_platform_driver(dw_mci_hi3798cv200_driver); MODULE_DESCRIPTION("HiSilicon Hi3798CV200 Specific DW-MSHC Driver Extension"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS("platform:dwmmc_hi3798cv200"); diff --git a/drivers/mmc/host/dw_mmc-hi3798mv200.c b/drivers/mmc/host/dw_mmc-hi3798mv200.c new file mode 100644 index 000000000000..989ae8dda722 --- /dev/null +++ b/drivers/mmc/host/dw_mmc-hi3798mv200.c @@ -0,0 +1,251 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Modified from dw_mmc-hi3798cv200.c + * + * Copyright (c) 2024 Yang Xiwen + * Copyright (c) 2018 HiSilicon Technologies Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "dw_mmc.h" +#include "dw_mmc-pltfm.h" + +#define SDMMC_TUNING_CTRL 0x118 +#define SDMMC_TUNING_FIND_EDGE BIT(5) + +#define ALL_INT_CLR 0x1ffff + +/* DLL ctrl reg */ +#define SAP_DLL_CTRL_DLLMODE BIT(16) + +struct dw_mci_hi3798mv200_priv { + struct clk *sample_clk; + struct clk *drive_clk; + struct regmap *crg_reg; + u32 sap_dll_offset; + struct mmc_clk_phase_map phase_map; +}; + +static void dw_mci_hi3798mv200_set_ios(struct dw_mci *host, struct mmc_ios *ios) +{ + struct dw_mci_hi3798mv200_priv *priv = host->priv; + struct mmc_clk_phase phase = priv->phase_map.phase[ios->timing]; + u32 val; + + val = mci_readl(host, ENABLE_SHIFT); + if (ios->timing == MMC_TIMING_MMC_DDR52 + || ios->timing == MMC_TIMING_UHS_DDR50) + val |= SDMMC_ENABLE_PHASE; + else + val &= ~SDMMC_ENABLE_PHASE; + mci_writel(host, ENABLE_SHIFT, val); + + val = mci_readl(host, DDR_REG); + if (ios->timing == MMC_TIMING_MMC_HS400) + val |= SDMMC_DDR_HS400; + else + val &= ~SDMMC_DDR_HS400; + mci_writel(host, DDR_REG, val); + + if (clk_set_rate(host->ciu_clk, ios->clock)) + dev_warn(host->dev, "Failed to set rate to %u\n", ios->clock); + else + /* + * CLK_MUX_ROUND_NEAREST is enabled for this clock + * The actual clock rate is not what we set, but a rounded value + * so we should get the rate once again + */ + host->bus_hz = clk_get_rate(host->ciu_clk); + + if (phase.valid) { + clk_set_phase(priv->drive_clk, phase.out_deg); + clk_set_phase(priv->sample_clk, phase.in_deg); + } else { + dev_warn(host->dev, + "The phase entry for timing mode %d is missing in device tree.\n", + ios->timing); + } +} + +static inline int dw_mci_hi3798mv200_enable_tuning(struct dw_mci_slot *slot) +{ + struct dw_mci_hi3798mv200_priv *priv = slot->host->priv; + + return regmap_clear_bits(priv->crg_reg, priv->sap_dll_offset, SAP_DLL_CTRL_DLLMODE); +} + +static inline int dw_mci_hi3798mv200_disable_tuning(struct dw_mci_slot *slot) +{ + struct dw_mci_hi3798mv200_priv *priv = slot->host->priv; + + return regmap_set_bits(priv->crg_reg, priv->sap_dll_offset, SAP_DLL_CTRL_DLLMODE); +} + +static int dw_mci_hi3798mv200_execute_tuning_mix_mode(struct dw_mci_slot *slot, + u32 opcode) +{ + static const int degrees[] = { 0, 45, 90, 135, 180, 225, 270, 315 }; + struct dw_mci *host = slot->host; + struct dw_mci_hi3798mv200_priv *priv = host->priv; + int raise_point = -1, fall_point = -1, mid; + int err, prev_err = -1; + int found = 0; + int regval; + int i; + int ret; + + ret = dw_mci_hi3798mv200_enable_tuning(slot); + if (ret < 0) + return ret; + + for (i = 0; i < ARRAY_SIZE(degrees); i++) { + clk_set_phase(priv->sample_clk, degrees[i]); + mci_writel(host, RINTSTS, ALL_INT_CLR); + + /* + * HiSilicon implemented a tuning mechanism. + * It needs special interaction with the DLL. + * + * Treat edge(flip) found as an error too. + */ + err = mmc_send_tuning(slot->mmc, opcode, NULL); + regval = mci_readl(host, TUNING_CTRL); + if (err || (regval & SDMMC_TUNING_FIND_EDGE)) + err = 1; + else + found = 1; + + if (i > 0) { + if (err && !prev_err) + fall_point = i - 1; + if (!err && prev_err) + raise_point = i; + } + + if (raise_point != -1 && fall_point != -1) + goto tuning_out; + + prev_err = err; + err = 0; + } + +tuning_out: + ret = dw_mci_hi3798mv200_disable_tuning(slot); + if (ret < 0) + return ret; + + if (found) { + if (raise_point == -1) + raise_point = 0; + if (fall_point == -1) + fall_point = ARRAY_SIZE(degrees) - 1; + if (fall_point < raise_point) { + if ((raise_point + fall_point) > + (ARRAY_SIZE(degrees) - 1)) + mid = fall_point / 2; + else + mid = (raise_point + ARRAY_SIZE(degrees) - 1) / 2; + } else { + mid = (raise_point + fall_point) / 2; + } + + /* + * We don't care what timing we are tuning for, + * simply use the same phase for all timing needs tuning. + */ + priv->phase_map.phase[MMC_TIMING_MMC_HS200].in_deg = degrees[mid]; + priv->phase_map.phase[MMC_TIMING_MMC_HS400].in_deg = degrees[mid]; + priv->phase_map.phase[MMC_TIMING_UHS_SDR104].in_deg = degrees[mid]; + + clk_set_phase(priv->sample_clk, degrees[mid]); + dev_dbg(host->dev, "Tuning clk_sample[%d, %d], set[%d]\n", + raise_point, fall_point, degrees[mid]); + ret = 0; + } else { + dev_err(host->dev, "No valid clk_sample shift!\n"); + ret = -EINVAL; + } + + mci_writel(host, RINTSTS, ALL_INT_CLR); + + return ret; +} + +static int dw_mci_hi3798mv200_init(struct dw_mci *host) +{ + struct dw_mci_hi3798mv200_priv *priv; + struct device_node *np = host->dev->of_node; + int ret; + + priv = devm_kzalloc(host->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + mmc_of_parse_clk_phase(host->dev, &priv->phase_map); + + priv->sample_clk = devm_clk_get_enabled(host->dev, "ciu-sample"); + if (IS_ERR(priv->sample_clk)) + return dev_err_probe(host->dev, PTR_ERR(priv->sample_clk), + "failed to get enabled ciu-sample clock\n"); + + priv->drive_clk = devm_clk_get_enabled(host->dev, "ciu-drive"); + if (IS_ERR(priv->drive_clk)) + return dev_err_probe(host->dev, PTR_ERR(priv->drive_clk), + "failed to get enabled ciu-drive clock\n"); + + priv->crg_reg = syscon_regmap_lookup_by_phandle(np, "hisilicon,sap-dll-reg"); + if (IS_ERR(priv->crg_reg)) + return dev_err_probe(host->dev, PTR_ERR(priv->crg_reg), + "failed to get CRG reg\n"); + + ret = of_property_read_u32_index(np, "hisilicon,sap-dll-reg", 1, &priv->sap_dll_offset); + if (ret) + return dev_err_probe(host->dev, ret, "failed to get sample DLL register offset\n"); + + host->priv = priv; + return 0; +} + +static const struct dw_mci_drv_data hi3798mv200_data = { + .common_caps = MMC_CAP_CMD23, + .init = dw_mci_hi3798mv200_init, + .set_ios = dw_mci_hi3798mv200_set_ios, + .execute_tuning = dw_mci_hi3798mv200_execute_tuning_mix_mode, +}; + +static const struct of_device_id dw_mci_hi3798mv200_match[] = { + { .compatible = "hisilicon,hi3798mv200-dw-mshc" }, + {}, +}; + +static int dw_mci_hi3798mv200_probe(struct platform_device *pdev) +{ + return dw_mci_pltfm_register(pdev, &hi3798mv200_data); +} + +static void dw_mci_hi3798mv200_remove(struct platform_device *pdev) +{ + dw_mci_pltfm_remove(pdev); +} + +MODULE_DEVICE_TABLE(of, dw_mci_hi3798mv200_match); +static struct platform_driver dw_mci_hi3798mv200_driver = { + .probe = dw_mci_hi3798mv200_probe, + .remove_new = dw_mci_hi3798mv200_remove, + .driver = { + .name = "dwmmc_hi3798mv200", + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + .of_match_table = dw_mci_hi3798mv200_match, + }, +}; +module_platform_driver(dw_mci_hi3798mv200_driver); + +MODULE_DESCRIPTION("HiSilicon Hi3798MV200 Specific DW-MSHC Driver Extension"); +MODULE_LICENSE("GPL"); diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c index 829af2c98a44..8e2d676b9239 100644 --- a/drivers/mmc/host/dw_mmc.c +++ b/drivers/mmc/host/dw_mmc.c @@ -35,7 +35,6 @@ #include #include #include -#include #include #include "dw_mmc.h" diff --git a/drivers/mmc/host/meson-mx-sdhc-clkc.c b/drivers/mmc/host/meson-mx-sdhc-clkc.c index 19200b7079a6..cbd17a596cd2 100644 --- a/drivers/mmc/host/meson-mx-sdhc-clkc.c +++ b/drivers/mmc/host/meson-mx-sdhc-clkc.c @@ -71,12 +71,23 @@ static int meson_mx_sdhc_clk_hw_register(struct device *dev, static int meson_mx_sdhc_gate_clk_hw_register(struct device *dev, const char *name_suffix, struct clk_hw *parent, - struct clk_hw *hw) + struct clk_hw *hw, + struct clk_bulk_data *clk_bulk_data, + u8 bulk_index) { struct clk_parent_data parent_data = { .hw = parent }; + int ret; - return meson_mx_sdhc_clk_hw_register(dev, name_suffix, &parent_data, 1, - &clk_gate_ops, hw); + ret = meson_mx_sdhc_clk_hw_register(dev, name_suffix, &parent_data, 1, + &clk_gate_ops, hw); + if (ret) + return ret; + + clk_bulk_data[bulk_index].clk = devm_clk_hw_get_clk(dev, hw, name_suffix); + if (IS_ERR(clk_bulk_data[bulk_index].clk)) + return PTR_ERR(clk_bulk_data[bulk_index].clk); + + return 0; } int meson_mx_sdhc_register_clkc(struct device *dev, void __iomem *base, @@ -115,7 +126,8 @@ int meson_mx_sdhc_register_clkc(struct device *dev, void __iomem *base, clkc_data->mod_clk_en.bit_idx = 15; ret = meson_mx_sdhc_gate_clk_hw_register(dev, "mod_clk_on", &clkc_data->div.hw, - &clkc_data->mod_clk_en.hw); + &clkc_data->mod_clk_en.hw, + clk_bulk_data, 0); if (ret) return ret; @@ -123,7 +135,8 @@ int meson_mx_sdhc_register_clkc(struct device *dev, void __iomem *base, clkc_data->tx_clk_en.bit_idx = 14; ret = meson_mx_sdhc_gate_clk_hw_register(dev, "tx_clk_on", &clkc_data->div.hw, - &clkc_data->tx_clk_en.hw); + &clkc_data->tx_clk_en.hw, + clk_bulk_data, 1); if (ret) return ret; @@ -131,7 +144,8 @@ int meson_mx_sdhc_register_clkc(struct device *dev, void __iomem *base, clkc_data->rx_clk_en.bit_idx = 13; ret = meson_mx_sdhc_gate_clk_hw_register(dev, "rx_clk_on", &clkc_data->div.hw, - &clkc_data->rx_clk_en.hw); + &clkc_data->rx_clk_en.hw, + clk_bulk_data, 2); if (ret) return ret; @@ -139,18 +153,7 @@ int meson_mx_sdhc_register_clkc(struct device *dev, void __iomem *base, clkc_data->sd_clk_en.bit_idx = 12; ret = meson_mx_sdhc_gate_clk_hw_register(dev, "sd_clk_on", &clkc_data->div.hw, - &clkc_data->sd_clk_en.hw); - if (ret) - return ret; - - /* - * TODO: Replace clk_hw.clk with devm_clk_hw_get_clk() once that is - * available. - */ - clk_bulk_data[0].clk = clkc_data->mod_clk_en.hw.clk; - clk_bulk_data[1].clk = clkc_data->sd_clk_en.hw.clk; - clk_bulk_data[2].clk = clkc_data->tx_clk_en.hw.clk; - clk_bulk_data[3].clk = clkc_data->rx_clk_en.hw.clk; - - return 0; + &clkc_data->sd_clk_en.hw, + clk_bulk_data, 3); + return ret; } diff --git a/drivers/mmc/host/meson-mx-sdhc-mmc.c b/drivers/mmc/host/meson-mx-sdhc-mmc.c index 1ed9731e77ef..31f750301dc1 100644 --- a/drivers/mmc/host/meson-mx-sdhc-mmc.c +++ b/drivers/mmc/host/meson-mx-sdhc-mmc.c @@ -65,10 +65,8 @@ static const struct regmap_config meson_mx_sdhc_regmap_config = { .max_register = MESON_SDHC_CLK2, }; -static void meson_mx_sdhc_hw_reset(struct mmc_host *mmc) +static void meson_mx_sdhc_reset(struct meson_mx_sdhc_host *host) { - struct meson_mx_sdhc_host *host = mmc_priv(mmc); - regmap_write(host->regmap, MESON_SDHC_SRST, MESON_SDHC_SRST_MAIN_CTRL | MESON_SDHC_SRST_RXFIFO | MESON_SDHC_SRST_TXFIFO | MESON_SDHC_SRST_DPHY_RX | MESON_SDHC_SRST_DPHY_TX | @@ -116,7 +114,7 @@ static void meson_mx_sdhc_wait_cmd_ready(struct mmc_host *mmc) dev_warn(mmc_dev(mmc), "Failed to poll for CMD_BUSY while processing CMD%d\n", host->cmd->opcode); - meson_mx_sdhc_hw_reset(mmc); + meson_mx_sdhc_reset(host); } ret = regmap_read_poll_timeout(host->regmap, MESON_SDHC_ESTA, esta, @@ -127,7 +125,7 @@ static void meson_mx_sdhc_wait_cmd_ready(struct mmc_host *mmc) dev_warn(mmc_dev(mmc), "Failed to poll for ESTA[13:11] while processing CMD%d\n", host->cmd->opcode); - meson_mx_sdhc_hw_reset(mmc); + meson_mx_sdhc_reset(host); } } @@ -495,7 +493,6 @@ static int meson_mx_sdhc_execute_tuning(struct mmc_host *mmc, u32 opcode) } static const struct mmc_host_ops meson_mx_sdhc_ops = { - .card_hw_reset = meson_mx_sdhc_hw_reset, .request = meson_mx_sdhc_request, .set_ios = meson_mx_sdhc_set_ios, .card_busy = meson_mx_sdhc_card_busy, @@ -618,7 +615,7 @@ static irqreturn_t meson_mx_sdhc_irq_thread(int irq, void *irq_data) } if (cmd->error == -EIO || cmd->error == -ETIMEDOUT) - meson_mx_sdhc_hw_reset(host->mmc); + meson_mx_sdhc_reset(host); else if (cmd->data) /* * Clear the FIFOs after completing data transfers to prevent @@ -728,7 +725,7 @@ static void meson_mx_sdhc_init_hw(struct mmc_host *mmc) { struct meson_mx_sdhc_host *host = mmc_priv(mmc); - meson_mx_sdhc_hw_reset(mmc); + meson_mx_sdhc_reset(host); regmap_write(host->regmap, MESON_SDHC_CTRL, FIELD_PREP(MESON_SDHC_CTRL_RX_PERIOD, 0xf) | diff --git a/drivers/mmc/host/mmc_spi.c b/drivers/mmc/host/mmc_spi.c index b8dda8160c4e..09d7a6a0dc1a 100644 --- a/drivers/mmc/host/mmc_spi.c +++ b/drivers/mmc/host/mmc_spi.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -510,10 +509,7 @@ mmc_spi_command_send(struct mmc_spi_host *host, * so we explicitly initialize it to all ones on RX paths. */ static void -mmc_spi_setup_data_message( - struct mmc_spi_host *host, - bool multiple, - enum dma_data_direction direction) +mmc_spi_setup_data_message(struct mmc_spi_host *host, bool multiple, bool write) { struct spi_transfer *t; struct scratch *scratch = host->data; @@ -523,7 +519,7 @@ mmc_spi_setup_data_message( /* for reads, readblock() skips 0xff bytes before finding * the token; for writes, this transfer issues that token. */ - if (direction == DMA_TO_DEVICE) { + if (write) { t = &host->token; memset(t, 0, sizeof(*t)); t->len = 1; @@ -547,7 +543,7 @@ mmc_spi_setup_data_message( t = &host->crc; memset(t, 0, sizeof(*t)); t->len = 2; - if (direction == DMA_TO_DEVICE) { + if (write) { /* the actual CRC may get written later */ t->tx_buf = &scratch->crc_val; } else { @@ -570,10 +566,10 @@ mmc_spi_setup_data_message( * the next token (next data block, or STOP_TRAN). We can try to * minimize I/O ops by using a single read to collect end-of-busy. */ - if (multiple || direction == DMA_TO_DEVICE) { + if (multiple || write) { t = &host->early_status; memset(t, 0, sizeof(*t)); - t->len = (direction == DMA_TO_DEVICE) ? sizeof(scratch->status) : 1; + t->len = write ? sizeof(scratch->status) : 1; t->tx_buf = host->ones; t->rx_buf = scratch->status; t->cs_change = 1; @@ -777,15 +773,15 @@ mmc_spi_data_do(struct mmc_spi_host *host, struct mmc_command *cmd, { struct spi_device *spi = host->spi; struct spi_transfer *t; - enum dma_data_direction direction = mmc_get_dma_dir(data); struct scatterlist *sg; unsigned n_sg; bool multiple = (data->blocks > 1); - const char *write_or_read = (direction == DMA_TO_DEVICE) ? "write" : "read"; + bool write = (data->flags & MMC_DATA_WRITE); + const char *write_or_read = write ? "write" : "read"; u32 clock_rate; unsigned long timeout; - mmc_spi_setup_data_message(host, multiple, direction); + mmc_spi_setup_data_message(host, multiple, write); t = &host->t; if (t->speed_hz) @@ -807,7 +803,7 @@ mmc_spi_data_do(struct mmc_spi_host *host, struct mmc_command *cmd, /* allow pio too; we don't allow highmem */ kmap_addr = kmap(sg_page(sg)); - if (direction == DMA_TO_DEVICE) + if (write) t->tx_buf = kmap_addr + sg->offset; else t->rx_buf = kmap_addr + sg->offset; @@ -818,7 +814,7 @@ mmc_spi_data_do(struct mmc_spi_host *host, struct mmc_command *cmd, dev_dbg(&spi->dev, " %s block, %d bytes\n", write_or_read, t->len); - if (direction == DMA_TO_DEVICE) + if (write) status = mmc_spi_writeblock(host, t, timeout); else status = mmc_spi_readblock(host, t, timeout); @@ -833,7 +829,9 @@ mmc_spi_data_do(struct mmc_spi_host *host, struct mmc_command *cmd, } /* discard mappings */ - if (direction == DMA_FROM_DEVICE) + if (write) + /* nothing to do */; + else flush_dcache_page(sg_page(sg)); kunmap(sg_page(sg)); @@ -850,7 +848,7 @@ mmc_spi_data_do(struct mmc_spi_host *host, struct mmc_command *cmd, * that can affect the STOP_TRAN logic. Complete (and current) * MMC specs should sort that out before Linux starts using CMD23. */ - if (direction == DMA_TO_DEVICE && multiple) { + if (write && multiple) { struct scratch *scratch = host->data; int tmp; const unsigned statlen = sizeof(scratch->status); @@ -935,7 +933,7 @@ static void mmc_spi_request(struct mmc_host *mmc, struct mmc_request *mrq) #endif /* request exclusive bus access */ - spi_bus_lock(host->spi->master); + spi_bus_lock(host->spi->controller); crc_recover: /* issue command; then optionally data and stop */ @@ -967,7 +965,7 @@ crc_recover: } /* release the bus */ - spi_bus_unlock(host->spi->master); + spi_bus_unlock(host->spi->controller); mmc_request_done(host->mmc, mrq); } @@ -1157,7 +1155,7 @@ static int mmc_spi_probe(struct spi_device *spi) /* We rely on full duplex transfers, mostly to reduce * per-transfer overheads (by making fewer transfers). */ - if (spi->master->flags & SPI_CONTROLLER_HALF_DUPLEX) + if (spi->controller->flags & SPI_CONTROLLER_HALF_DUPLEX) return -EINVAL; /* MMC and SD specs only seem to care that sampling is on the diff --git a/drivers/mmc/host/moxart-mmc.c b/drivers/mmc/host/moxart-mmc.c index 5cfdd3a86e54..b88d6dec209f 100644 --- a/drivers/mmc/host/moxart-mmc.c +++ b/drivers/mmc/host/moxart-mmc.c @@ -131,12 +131,10 @@ struct moxart_host { struct dma_async_tx_descriptor *tx_desc; struct mmc_host *mmc; struct mmc_request *mrq; - struct scatterlist *cur_sg; struct completion dma_complete; struct completion pio_complete; - u32 num_sg; - u32 data_remain; + struct sg_mapping_iter sg_miter; u32 data_len; u32 fifo_width; u32 timeout; @@ -148,35 +146,6 @@ struct moxart_host { bool is_removed; }; -static inline void moxart_init_sg(struct moxart_host *host, - struct mmc_data *data) -{ - host->cur_sg = data->sg; - host->num_sg = data->sg_len; - host->data_remain = host->cur_sg->length; - - if (host->data_remain > host->data_len) - host->data_remain = host->data_len; -} - -static inline int moxart_next_sg(struct moxart_host *host) -{ - int remain; - struct mmc_data *data = host->mrq->cmd->data; - - host->cur_sg++; - host->num_sg--; - - if (host->num_sg > 0) { - host->data_remain = host->cur_sg->length; - remain = host->data_len - data->bytes_xfered; - if (remain > 0 && remain < host->data_remain) - host->data_remain = remain; - } - - return host->num_sg; -} - static int moxart_wait_for_status(struct moxart_host *host, u32 mask, u32 *status) { @@ -254,6 +223,11 @@ static void moxart_dma_complete(void *param) complete(&host->dma_complete); } +static bool moxart_use_dma(struct moxart_host *host) +{ + return (host->data_len > host->fifo_width) && host->have_dma; +} + static void moxart_transfer_dma(struct mmc_data *data, struct moxart_host *host) { u32 len, dir_slave; @@ -291,11 +265,11 @@ static void moxart_transfer_dma(struct mmc_data *data, struct moxart_host *host) dma_async_issue_pending(dma_chan); } - data->bytes_xfered += host->data_remain; - wait_for_completion_interruptible_timeout(&host->dma_complete, host->timeout); + data->bytes_xfered = host->data_len; + dma_unmap_sg(dma_chan->device->dev, data->sg, data->sg_len, mmc_get_dma_dir(data)); @@ -304,14 +278,28 @@ static void moxart_transfer_dma(struct mmc_data *data, struct moxart_host *host) static void moxart_transfer_pio(struct moxart_host *host) { + struct sg_mapping_iter *sgm = &host->sg_miter; struct mmc_data *data = host->mrq->cmd->data; u32 *sgp, len = 0, remain, status; if (host->data_len == data->bytes_xfered) return; - sgp = sg_virt(host->cur_sg); - remain = host->data_remain; + /* + * By updating sgm->consumes this will get a proper pointer into the + * buffer at any time. + */ + if (!sg_miter_next(sgm)) { + /* This shold not happen */ + dev_err(mmc_dev(host->mmc), "ran out of scatterlist prematurely\n"); + data->error = -EINVAL; + complete(&host->pio_complete); + return; + } + sgp = sgm->addr; + remain = sgm->length; + if (remain > host->data_len) + remain = host->data_len; if (data->flags & MMC_DATA_WRITE) { while (remain > 0) { @@ -326,6 +314,7 @@ static void moxart_transfer_pio(struct moxart_host *host) sgp++; len += 4; } + sgm->consumed += len; remain -= len; } @@ -342,22 +331,22 @@ static void moxart_transfer_pio(struct moxart_host *host) sgp++; len += 4; } + sgm->consumed += len; remain -= len; } } - data->bytes_xfered += host->data_remain - remain; - host->data_remain = remain; - - if (host->data_len != data->bytes_xfered) - moxart_next_sg(host); - else + data->bytes_xfered += sgm->consumed; + if (host->data_len == data->bytes_xfered) { complete(&host->pio_complete); + return; + } } static void moxart_prepare_data(struct moxart_host *host) { struct mmc_data *data = host->mrq->cmd->data; + unsigned int flags = SG_MITER_ATOMIC; /* Used from IRQ */ u32 datactrl; int blksz_bits; @@ -368,15 +357,19 @@ static void moxart_prepare_data(struct moxart_host *host) blksz_bits = ffs(data->blksz) - 1; BUG_ON(1 << blksz_bits != data->blksz); - moxart_init_sg(host, data); - datactrl = DCR_DATA_EN | (blksz_bits & DCR_BLK_SIZE); - if (data->flags & MMC_DATA_WRITE) + if (data->flags & MMC_DATA_WRITE) { + flags |= SG_MITER_FROM_SG; datactrl |= DCR_DATA_WRITE; + } else { + flags |= SG_MITER_TO_SG; + } - if ((host->data_len > host->fifo_width) && host->have_dma) + if (moxart_use_dma(host)) datactrl |= DCR_DMA_EN; + else + sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags); writel(DCR_DATA_FIFO_RESET, host->base + REG_DATA_CONTROL); writel(MASK_DATA | FIFO_URUN | FIFO_ORUN, host->base + REG_CLEAR); @@ -407,7 +400,7 @@ static void moxart_request(struct mmc_host *mmc, struct mmc_request *mrq) moxart_send_command(host, host->mrq->cmd); if (mrq->cmd->data) { - if ((host->data_len > host->fifo_width) && host->have_dma) { + if (moxart_use_dma(host)) { writel(CARD_CHANGE, host->base + REG_INTERRUPT_MASK); @@ -449,6 +442,9 @@ static void moxart_request(struct mmc_host *mmc, struct mmc_request *mrq) } request_done: + if (!moxart_use_dma(host)) + sg_miter_stop(&host->sg_miter); + spin_unlock_irqrestore(&host->lock, flags); mmc_request_done(host->mmc, mrq); } diff --git a/drivers/mmc/host/mvsdio.c b/drivers/mmc/host/mvsdio.c index ca01b7d204ba..af7f21888e27 100644 --- a/drivers/mmc/host/mvsdio.c +++ b/drivers/mmc/host/mvsdio.c @@ -38,8 +38,9 @@ struct mvsd_host { unsigned int xfer_mode; unsigned int intr_en; unsigned int ctrl; + bool use_pio; + struct sg_mapping_iter sg_miter; unsigned int pio_size; - void *pio_ptr; unsigned int sg_frags; unsigned int ns_per_clk; unsigned int clock; @@ -114,11 +115,18 @@ static int mvsd_setup_data(struct mvsd_host *host, struct mmc_data *data) * data when the buffer is not aligned on a 64 byte * boundary. */ + unsigned int miter_flags = SG_MITER_ATOMIC; /* Used from IRQ */ + + if (data->flags & MMC_DATA_READ) + miter_flags |= SG_MITER_TO_SG; + else + miter_flags |= SG_MITER_FROM_SG; + host->pio_size = data->blocks * data->blksz; - host->pio_ptr = sg_virt(data->sg); + sg_miter_start(&host->sg_miter, data->sg, data->sg_len, miter_flags); if (!nodma) - dev_dbg(host->dev, "fallback to PIO for data at 0x%p size %d\n", - host->pio_ptr, host->pio_size); + dev_dbg(host->dev, "fallback to PIO for data\n"); + host->use_pio = true; return 1; } else { dma_addr_t phys_addr; @@ -129,6 +137,7 @@ static int mvsd_setup_data(struct mvsd_host *host, struct mmc_data *data) phys_addr = sg_dma_address(data->sg); mvsd_write(MVSD_SYS_ADDR_LOW, (u32)phys_addr & 0xffff); mvsd_write(MVSD_SYS_ADDR_HI, (u32)phys_addr >> 16); + host->use_pio = false; return 0; } } @@ -288,8 +297,8 @@ static u32 mvsd_finish_data(struct mvsd_host *host, struct mmc_data *data, { void __iomem *iobase = host->base; - if (host->pio_ptr) { - host->pio_ptr = NULL; + if (host->use_pio) { + sg_miter_stop(&host->sg_miter); host->pio_size = 0; } else { dma_unmap_sg(mmc_dev(host->mmc), data->sg, host->sg_frags, @@ -344,9 +353,12 @@ static u32 mvsd_finish_data(struct mvsd_host *host, struct mmc_data *data, static irqreturn_t mvsd_irq(int irq, void *dev) { struct mvsd_host *host = dev; + struct sg_mapping_iter *sgm = &host->sg_miter; void __iomem *iobase = host->base; u32 intr_status, intr_done_mask; int irq_handled = 0; + u16 *p; + int s; intr_status = mvsd_read(MVSD_NOR_INTR_STATUS); dev_dbg(host->dev, "intr 0x%04x intr_en 0x%04x hw_state 0x%04x\n", @@ -370,15 +382,36 @@ static irqreturn_t mvsd_irq(int irq, void *dev) spin_lock(&host->lock); /* PIO handling, if needed. Messy business... */ - if (host->pio_size && + if (host->use_pio) { + /* + * As we set sgm->consumed this always gives a valid buffer + * position. + */ + if (!sg_miter_next(sgm)) { + /* This should not happen */ + dev_err(host->dev, "ran out of scatter segments\n"); + spin_unlock(&host->lock); + host->intr_en &= + ~(MVSD_NOR_RX_READY | MVSD_NOR_RX_FIFO_8W | + MVSD_NOR_TX_AVAIL | MVSD_NOR_TX_FIFO_8W); + mvsd_write(MVSD_NOR_INTR_EN, host->intr_en); + return IRQ_HANDLED; + } + p = sgm->addr; + s = sgm->length; + if (s > host->pio_size) + s = host->pio_size; + } + + if (host->use_pio && (intr_status & host->intr_en & (MVSD_NOR_RX_READY | MVSD_NOR_RX_FIFO_8W))) { - u16 *p = host->pio_ptr; - int s = host->pio_size; + while (s >= 32 && (intr_status & MVSD_NOR_RX_FIFO_8W)) { readsw(iobase + MVSD_FIFO, p, 16); p += 16; s -= 32; + sgm->consumed += 32; intr_status = mvsd_read(MVSD_NOR_INTR_STATUS); } /* @@ -391,6 +424,7 @@ static irqreturn_t mvsd_irq(int irq, void *dev) put_unaligned(mvsd_read(MVSD_FIFO), p++); put_unaligned(mvsd_read(MVSD_FIFO), p++); s -= 4; + sgm->consumed += 4; intr_status = mvsd_read(MVSD_NOR_INTR_STATUS); } if (s && s < 4 && (intr_status & MVSD_NOR_RX_READY)) { @@ -398,10 +432,13 @@ static irqreturn_t mvsd_irq(int irq, void *dev) val[0] = mvsd_read(MVSD_FIFO); val[1] = mvsd_read(MVSD_FIFO); memcpy(p, ((void *)&val) + 4 - s, s); + sgm->consumed += s; s = 0; intr_status = mvsd_read(MVSD_NOR_INTR_STATUS); } - if (s == 0) { + /* PIO transfer done */ + host->pio_size -= sgm->consumed; + if (host->pio_size == 0) { host->intr_en &= ~(MVSD_NOR_RX_READY | MVSD_NOR_RX_FIFO_8W); mvsd_write(MVSD_NOR_INTR_EN, host->intr_en); @@ -413,14 +450,10 @@ static irqreturn_t mvsd_irq(int irq, void *dev) } dev_dbg(host->dev, "pio %d intr 0x%04x hw_state 0x%04x\n", s, intr_status, mvsd_read(MVSD_HW_STATE)); - host->pio_ptr = p; - host->pio_size = s; irq_handled = 1; - } else if (host->pio_size && + } else if (host->use_pio && (intr_status & host->intr_en & (MVSD_NOR_TX_AVAIL | MVSD_NOR_TX_FIFO_8W))) { - u16 *p = host->pio_ptr; - int s = host->pio_size; /* * The TX_FIFO_8W bit is unreliable. When set, bursting * 16 halfwords all at once in the FIFO drops data. Actually @@ -431,6 +464,7 @@ static irqreturn_t mvsd_irq(int irq, void *dev) mvsd_write(MVSD_FIFO, get_unaligned(p++)); mvsd_write(MVSD_FIFO, get_unaligned(p++)); s -= 4; + sgm->consumed += 4; intr_status = mvsd_read(MVSD_NOR_INTR_STATUS); } if (s < 4) { @@ -439,10 +473,13 @@ static irqreturn_t mvsd_irq(int irq, void *dev) memcpy(((void *)&val) + 4 - s, p, s); mvsd_write(MVSD_FIFO, val[0]); mvsd_write(MVSD_FIFO, val[1]); + sgm->consumed += s; s = 0; intr_status = mvsd_read(MVSD_NOR_INTR_STATUS); } - if (s == 0) { + /* PIO transfer done */ + host->pio_size -= sgm->consumed; + if (host->pio_size == 0) { host->intr_en &= ~(MVSD_NOR_TX_AVAIL | MVSD_NOR_TX_FIFO_8W); mvsd_write(MVSD_NOR_INTR_EN, host->intr_en); @@ -450,8 +487,6 @@ static irqreturn_t mvsd_irq(int irq, void *dev) } dev_dbg(host->dev, "pio %d intr 0x%04x hw_state 0x%04x\n", s, intr_status, mvsd_read(MVSD_HW_STATE)); - host->pio_ptr = p; - host->pio_size = s; irq_handled = 1; } diff --git a/drivers/mmc/host/mxcmmc.c b/drivers/mmc/host/mxcmmc.c index 5b3ab0e20505..1edf65291354 100644 --- a/drivers/mmc/host/mxcmmc.c +++ b/drivers/mmc/host/mxcmmc.c @@ -266,11 +266,18 @@ static inline void buffer_swap32(u32 *buf, int len) static void mxcmci_swap_buffers(struct mmc_data *data) { - struct scatterlist *sg; - int i; + struct sg_mapping_iter sgm; + u32 *buf; - for_each_sg(data->sg, sg, data->sg_len, i) - buffer_swap32(sg_virt(sg), sg->length); + sg_miter_start(&sgm, data->sg, data->sg_len, + SG_MITER_TO_SG | SG_MITER_FROM_SG); + + while (sg_miter_next(&sgm)) { + buf = sgm.addr; + buffer_swap32(buf, sgm.length); + } + + sg_miter_stop(&sgm); } #else static inline void mxcmci_swap_buffers(struct mmc_data *data) {} @@ -526,10 +533,9 @@ static int mxcmci_poll_status(struct mxcmci_host *host, u32 mask) } while (1); } -static int mxcmci_pull(struct mxcmci_host *host, void *_buf, int bytes) +static int mxcmci_pull(struct mxcmci_host *host, u32 *buf, int bytes) { unsigned int stat; - u32 *buf = _buf; while (bytes > 3) { stat = mxcmci_poll_status(host, @@ -555,10 +561,9 @@ static int mxcmci_pull(struct mxcmci_host *host, void *_buf, int bytes) return 0; } -static int mxcmci_push(struct mxcmci_host *host, void *_buf, int bytes) +static int mxcmci_push(struct mxcmci_host *host, u32 *buf, int bytes) { unsigned int stat; - u32 *buf = _buf; while (bytes > 3) { stat = mxcmci_poll_status(host, STATUS_BUF_WRITE_RDY); @@ -586,31 +591,39 @@ static int mxcmci_push(struct mxcmci_host *host, void *_buf, int bytes) static int mxcmci_transfer_data(struct mxcmci_host *host) { struct mmc_data *data = host->req->data; - struct scatterlist *sg; - int stat, i; + struct sg_mapping_iter sgm; + int stat; + u32 *buf; host->data = data; host->datasize = 0; + sg_miter_start(&sgm, data->sg, data->sg_len, + (data->flags & MMC_DATA_READ) ? SG_MITER_TO_SG : SG_MITER_FROM_SG); if (data->flags & MMC_DATA_READ) { - for_each_sg(data->sg, sg, data->sg_len, i) { - stat = mxcmci_pull(host, sg_virt(sg), sg->length); + while (sg_miter_next(&sgm)) { + buf = sgm.addr; + stat = mxcmci_pull(host, buf, sgm.length); if (stat) - return stat; - host->datasize += sg->length; + goto transfer_error; + host->datasize += sgm.length; } } else { - for_each_sg(data->sg, sg, data->sg_len, i) { - stat = mxcmci_push(host, sg_virt(sg), sg->length); + while (sg_miter_next(&sgm)) { + buf = sgm.addr; + stat = mxcmci_push(host, buf, sgm.length); if (stat) - return stat; - host->datasize += sg->length; + goto transfer_error; + host->datasize += sgm.length; } stat = mxcmci_poll_status(host, STATUS_WRITE_OP_DONE); if (stat) - return stat; + goto transfer_error; } - return 0; + +transfer_error: + sg_miter_stop(&sgm); + return stat; } static void mxcmci_datawork(struct work_struct *work) diff --git a/drivers/mmc/host/omap.c b/drivers/mmc/host/omap.c index 9fb8995b43a1..088f8ed4fdc4 100644 --- a/drivers/mmc/host/omap.c +++ b/drivers/mmc/host/omap.c @@ -148,10 +148,8 @@ struct mmc_omap_host { struct work_struct send_stop_work; struct mmc_data *stop_data; + struct sg_mapping_iter sg_miter; unsigned int sg_len; - int sg_idx; - u16 * buffer; - u32 buffer_bytes_left; u32 total_bytes_left; unsigned features; @@ -456,6 +454,8 @@ mmc_omap_xfer_done(struct mmc_omap_host *host, struct mmc_data *data) { if (host->dma_in_use) mmc_omap_release_dma(host, data, data->error); + else + sg_miter_stop(&host->sg_miter); host->data = NULL; host->sg_len = 0; @@ -651,19 +651,6 @@ mmc_omap_cmd_timer(struct timer_list *t) spin_unlock_irqrestore(&host->slot_lock, flags); } -/* PIO only */ -static void -mmc_omap_sg_to_buf(struct mmc_omap_host *host) -{ - struct scatterlist *sg; - - sg = host->data->sg + host->sg_idx; - host->buffer_bytes_left = sg->length; - host->buffer = sg_virt(sg); - if (host->buffer_bytes_left > host->total_bytes_left) - host->buffer_bytes_left = host->total_bytes_left; -} - static void mmc_omap_clk_timer(struct timer_list *t) { @@ -676,33 +663,37 @@ mmc_omap_clk_timer(struct timer_list *t) static void mmc_omap_xfer_data(struct mmc_omap_host *host, int write) { + struct sg_mapping_iter *sgm = &host->sg_miter; int n, nwords; + u16 *buffer; - if (host->buffer_bytes_left == 0) { - host->sg_idx++; - BUG_ON(host->sg_idx == host->sg_len); - mmc_omap_sg_to_buf(host); + if (!sg_miter_next(sgm)) { + /* This should not happen */ + dev_err(mmc_dev(host->mmc), "ran out of scatterlist prematurely\n"); + return; } + buffer = sgm->addr; + n = 64; - if (n > host->buffer_bytes_left) - n = host->buffer_bytes_left; + if (n > sgm->length) + n = sgm->length; + if (n > host->total_bytes_left) + n = host->total_bytes_left; /* Round up to handle odd number of bytes to transfer */ nwords = DIV_ROUND_UP(n, 2); - host->buffer_bytes_left -= n; + sgm->consumed = n; host->total_bytes_left -= n; host->data->bytes_xfered += n; if (write) { __raw_writesw(host->virt_base + OMAP_MMC_REG(host, DATA), - host->buffer, nwords); + buffer, nwords); } else { __raw_readsw(host->virt_base + OMAP_MMC_REG(host, DATA), - host->buffer, nwords); + buffer, nwords); } - - host->buffer += nwords; } #ifdef CONFIG_MMC_DEBUG @@ -956,6 +947,7 @@ static inline void set_data_timeout(struct mmc_omap_host *host, struct mmc_reque static void mmc_omap_prepare_data(struct mmc_omap_host *host, struct mmc_request *req) { + unsigned int miter_flags = SG_MITER_ATOMIC; /* Used from IRQ */ struct mmc_data *data = req->data; int i, use_dma = 1, block_size; struct scatterlist *sg; @@ -990,7 +982,6 @@ mmc_omap_prepare_data(struct mmc_omap_host *host, struct mmc_request *req) } } - host->sg_idx = 0; if (use_dma) { enum dma_data_direction dma_data_dir; struct dma_async_tx_descriptor *tx; @@ -1071,7 +1062,11 @@ mmc_omap_prepare_data(struct mmc_omap_host *host, struct mmc_request *req) OMAP_MMC_WRITE(host, BUF, 0x1f1f); host->total_bytes_left = data->blocks * block_size; host->sg_len = sg_len; - mmc_omap_sg_to_buf(host); + if (data->flags & MMC_DATA_READ) + miter_flags |= SG_MITER_TO_SG; + else + miter_flags |= SG_MITER_FROM_SG; + sg_miter_start(&host->sg_miter, data->sg, data->sg_len, miter_flags); host->dma_in_use = 0; } diff --git a/drivers/mmc/host/renesas_sdhi.h b/drivers/mmc/host/renesas_sdhi.h index c1fb9740eab0..586f94d4dbfd 100644 --- a/drivers/mmc/host/renesas_sdhi.h +++ b/drivers/mmc/host/renesas_sdhi.h @@ -9,6 +9,7 @@ #ifndef RENESAS_SDHI_H #define RENESAS_SDHI_H +#include #include #include "tmio_mmc.h" @@ -63,7 +64,7 @@ struct renesas_sdhi_of_data_with_quirks { struct renesas_sdhi_dma { unsigned long end_flags; enum dma_slave_buswidth dma_buswidth; - bool (*filter)(struct dma_chan *chan, void *arg); + dma_filter_fn filter; void (*enable)(struct tmio_mmc_host *host, bool enable); struct completion dma_dataend; struct tasklet_struct dma_complete; diff --git a/drivers/mmc/host/sdhci-esdhc-mcf.c b/drivers/mmc/host/sdhci-esdhc-mcf.c index a07f8333cd6b..c97363e2d86c 100644 --- a/drivers/mmc/host/sdhci-esdhc-mcf.c +++ b/drivers/mmc/host/sdhci-esdhc-mcf.c @@ -299,9 +299,8 @@ static void esdhc_mcf_pltfm_set_bus_width(struct sdhci_host *host, int width) static void esdhc_mcf_request_done(struct sdhci_host *host, struct mmc_request *mrq) { - struct scatterlist *sg; + struct sg_mapping_iter sgm; u32 *buffer; - int i; if (!mrq->data || !mrq->data->bytes_xfered) goto exit_done; @@ -313,10 +312,13 @@ static void esdhc_mcf_request_done(struct sdhci_host *host, * On mcf5441x there is no hw sdma option/flag to select the dma * transfer endiannes. A swap after the transfer is needed. */ - for_each_sg(mrq->data->sg, sg, mrq->data->sg_len, i) { - buffer = (u32 *)sg_virt(sg); - esdhc_mcf_buffer_swap32(buffer, sg->length); + sg_miter_start(&sgm, mrq->data->sg, mrq->data->sg_len, + SG_MITER_ATOMIC | SG_MITER_TO_SG | SG_MITER_FROM_SG); + while (sg_miter_next(&sgm)) { + buffer = sgm.addr; + esdhc_mcf_buffer_swap32(buffer, sgm.length); } + sg_miter_stop(&sgm); exit_done: mmc_request_done(host->mmc, mrq); diff --git a/drivers/mmc/host/sdhci-of-aspeed.c b/drivers/mmc/host/sdhci-of-aspeed.c index 42d54532cabe..430c1f90037b 100644 --- a/drivers/mmc/host/sdhci-of-aspeed.c +++ b/drivers/mmc/host/sdhci-of-aspeed.c @@ -435,7 +435,7 @@ static int aspeed_sdhci_probe(struct platform_device *pdev) goto err_sdhci_add; if (dev->phase_desc) - mmc_of_parse_clk_phase(host->mmc, &dev->phase_map); + mmc_of_parse_clk_phase(&pdev->dev, &dev->phase_map); ret = sdhci_add_host(host); if (ret) diff --git a/drivers/mmc/host/sdhci-of-dwcmshc.c b/drivers/mmc/host/sdhci-of-dwcmshc.c index a1f57af6acfb..ab4b964d4058 100644 --- a/drivers/mmc/host/sdhci-of-dwcmshc.c +++ b/drivers/mmc/host/sdhci-of-dwcmshc.c @@ -52,6 +52,20 @@ #define AT_CTRL_SWIN_TH_VAL_MASK GENMASK(31, 24) /* bits [31:24] */ #define AT_CTRL_SWIN_TH_VAL 0x9 /* sampling window threshold */ +/* Sophgo CV18XX specific Registers */ +#define CV18XX_SDHCI_MSHC_CTRL 0x00 +#define CV18XX_EMMC_FUNC_EN BIT(0) +#define CV18XX_LATANCY_1T BIT(1) +#define CV18XX_SDHCI_PHY_TX_RX_DLY 0x40 +#define CV18XX_PHY_TX_DLY_MSK GENMASK(6, 0) +#define CV18XX_PHY_TX_SRC_MSK GENMASK(9, 8) +#define CV18XX_PHY_TX_SRC_INVERT_CLK_TX 0x1 +#define CV18XX_PHY_RX_DLY_MSK GENMASK(22, 16) +#define CV18XX_PHY_RX_SRC_MSK GENMASK(25, 24) +#define CV18XX_PHY_RX_SRC_INVERT_RX_CLK 0x1 +#define CV18XX_SDHCI_PHY_CONFIG 0x4c +#define CV18XX_PHY_TX_BPS BIT(0) + /* Rockchip specific Registers */ #define DWCMSHC_EMMC_DLL_CTRL 0x800 #define DWCMSHC_EMMC_DLL_RXCLK 0x804 @@ -642,6 +656,35 @@ static void th1520_sdhci_reset(struct sdhci_host *host, u8 mask) } } +static void cv18xx_sdhci_reset(struct sdhci_host *host, u8 mask) +{ + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct dwcmshc_priv *priv = sdhci_pltfm_priv(pltfm_host); + u32 val, emmc_caps = MMC_CAP2_NO_SD | MMC_CAP2_NO_SDIO; + + sdhci_reset(host, mask); + + if ((host->mmc->caps2 & emmc_caps) == emmc_caps) { + val = sdhci_readl(host, priv->vendor_specific_area1 + CV18XX_SDHCI_MSHC_CTRL); + val |= CV18XX_EMMC_FUNC_EN; + sdhci_writel(host, val, priv->vendor_specific_area1 + CV18XX_SDHCI_MSHC_CTRL); + } + + val = sdhci_readl(host, priv->vendor_specific_area1 + CV18XX_SDHCI_MSHC_CTRL); + val |= CV18XX_LATANCY_1T; + sdhci_writel(host, val, priv->vendor_specific_area1 + CV18XX_SDHCI_MSHC_CTRL); + + val = sdhci_readl(host, priv->vendor_specific_area1 + CV18XX_SDHCI_PHY_CONFIG); + val |= CV18XX_PHY_TX_BPS; + sdhci_writel(host, val, priv->vendor_specific_area1 + CV18XX_SDHCI_PHY_CONFIG); + + val = (FIELD_PREP(CV18XX_PHY_TX_DLY_MSK, 0) | + FIELD_PREP(CV18XX_PHY_TX_SRC_MSK, CV18XX_PHY_TX_SRC_INVERT_CLK_TX) | + FIELD_PREP(CV18XX_PHY_RX_DLY_MSK, 0) | + FIELD_PREP(CV18XX_PHY_RX_SRC_MSK, CV18XX_PHY_RX_SRC_INVERT_RX_CLK)); + sdhci_writel(host, val, priv->vendor_specific_area1 + CV18XX_SDHCI_PHY_TX_RX_DLY); +} + static const struct sdhci_ops sdhci_dwcmshc_ops = { .set_clock = sdhci_set_clock, .set_bus_width = sdhci_set_bus_width, @@ -671,6 +714,15 @@ static const struct sdhci_ops sdhci_dwcmshc_th1520_ops = { .platform_execute_tuning = &th1520_execute_tuning, }; +static const struct sdhci_ops sdhci_dwcmshc_cv18xx_ops = { + .set_clock = sdhci_set_clock, + .set_bus_width = sdhci_set_bus_width, + .set_uhs_signaling = dwcmshc_set_uhs_signaling, + .get_max_clock = dwcmshc_get_max_clock, + .reset = cv18xx_sdhci_reset, + .adma_write_desc = dwcmshc_adma_write_desc, +}; + static const struct sdhci_pltfm_data sdhci_dwcmshc_pdata = { .ops = &sdhci_dwcmshc_ops, .quirks = SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN, @@ -700,6 +752,12 @@ static const struct sdhci_pltfm_data sdhci_dwcmshc_th1520_pdata = { .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN, }; +static const struct sdhci_pltfm_data sdhci_dwcmshc_cv18xx_pdata = { + .ops = &sdhci_dwcmshc_cv18xx_ops, + .quirks = SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN, + .quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN, +}; + static int dwcmshc_rk35xx_init(struct sdhci_host *host, struct dwcmshc_priv *dwc_priv) { int err; @@ -768,6 +826,14 @@ static const struct of_device_id sdhci_dwcmshc_dt_ids[] = { .compatible = "snps,dwcmshc-sdhci", .data = &sdhci_dwcmshc_pdata, }, + { + .compatible = "sophgo,cv1800b-dwcmshc", + .data = &sdhci_dwcmshc_cv18xx_pdata, + }, + { + .compatible = "sophgo,sg2002-dwcmshc", + .data = &sdhci_dwcmshc_cv18xx_pdata, + }, { .compatible = "thead,th1520-dwcmshc", .data = &sdhci_dwcmshc_th1520_pdata, diff --git a/drivers/mmc/host/sh_mmcif.c b/drivers/mmc/host/sh_mmcif.c index 077d711e964e..08b4312af94e 100644 --- a/drivers/mmc/host/sh_mmcif.c +++ b/drivers/mmc/host/sh_mmcif.c @@ -227,14 +227,12 @@ struct sh_mmcif_host { bool dying; long timeout; void __iomem *addr; - u32 *pio_ptr; spinlock_t lock; /* protect sh_mmcif_host::state */ enum sh_mmcif_state state; enum sh_mmcif_wait_for wait_for; struct delayed_work timeout_work; size_t blocksize; - int sg_idx; - int sg_blkidx; + struct sg_mapping_iter sg_miter; bool power; bool ccs_enable; /* Command Completion Signal support */ bool clk_ctrl2_enable; @@ -600,32 +598,17 @@ static int sh_mmcif_error_manage(struct sh_mmcif_host *host) return ret; } -static bool sh_mmcif_next_block(struct sh_mmcif_host *host, u32 *p) -{ - struct mmc_data *data = host->mrq->data; - - host->sg_blkidx += host->blocksize; - - /* data->sg->length must be a multiple of host->blocksize? */ - BUG_ON(host->sg_blkidx > data->sg->length); - - if (host->sg_blkidx == data->sg->length) { - host->sg_blkidx = 0; - if (++host->sg_idx < data->sg_len) - host->pio_ptr = sg_virt(++data->sg); - } else { - host->pio_ptr = p; - } - - return host->sg_idx != data->sg_len; -} - static void sh_mmcif_single_read(struct sh_mmcif_host *host, struct mmc_request *mrq) { + struct mmc_data *data = mrq->data; + host->blocksize = (sh_mmcif_readl(host->addr, MMCIF_CE_BLOCK_SET) & BLOCK_SIZE_MASK) + 3; + sg_miter_start(&host->sg_miter, data->sg, data->sg_len, + SG_MITER_TO_SG); + host->wait_for = MMCIF_WAIT_FOR_READ; /* buf read enable */ @@ -634,20 +617,32 @@ static void sh_mmcif_single_read(struct sh_mmcif_host *host, static bool sh_mmcif_read_block(struct sh_mmcif_host *host) { + struct sg_mapping_iter *sgm = &host->sg_miter; struct device *dev = sh_mmcif_host_to_dev(host); struct mmc_data *data = host->mrq->data; - u32 *p = sg_virt(data->sg); + u32 *p; int i; if (host->sd_error) { + sg_miter_stop(sgm); data->error = sh_mmcif_error_manage(host); dev_dbg(dev, "%s(): %d\n", __func__, data->error); return false; } + if (!sg_miter_next(sgm)) { + /* This should not happen on single blocks */ + sg_miter_stop(sgm); + return false; + } + + p = sgm->addr; + for (i = 0; i < host->blocksize / 4; i++) *p++ = sh_mmcif_readl(host->addr, MMCIF_CE_DATA); + sg_miter_stop(&host->sg_miter); + /* buffer read end */ sh_mmcif_bitset(host, MMCIF_CE_INT_MASK, MASK_MBUFRE); host->wait_for = MMCIF_WAIT_FOR_READ_END; @@ -658,6 +653,7 @@ static bool sh_mmcif_read_block(struct sh_mmcif_host *host) static void sh_mmcif_multi_read(struct sh_mmcif_host *host, struct mmc_request *mrq) { + struct sg_mapping_iter *sgm = &host->sg_miter; struct mmc_data *data = mrq->data; if (!data->sg_len || !data->sg->length) @@ -666,46 +662,63 @@ static void sh_mmcif_multi_read(struct sh_mmcif_host *host, host->blocksize = sh_mmcif_readl(host->addr, MMCIF_CE_BLOCK_SET) & BLOCK_SIZE_MASK; + sg_miter_start(sgm, data->sg, data->sg_len, + SG_MITER_TO_SG); + + /* Advance to the first sglist entry */ + if (!sg_miter_next(sgm)) { + sg_miter_stop(sgm); + return; + } + host->wait_for = MMCIF_WAIT_FOR_MREAD; - host->sg_idx = 0; - host->sg_blkidx = 0; - host->pio_ptr = sg_virt(data->sg); sh_mmcif_bitset(host, MMCIF_CE_INT_MASK, MASK_MBUFREN); } static bool sh_mmcif_mread_block(struct sh_mmcif_host *host) { + struct sg_mapping_iter *sgm = &host->sg_miter; struct device *dev = sh_mmcif_host_to_dev(host); struct mmc_data *data = host->mrq->data; - u32 *p = host->pio_ptr; + u32 *p; int i; if (host->sd_error) { + sg_miter_stop(sgm); data->error = sh_mmcif_error_manage(host); dev_dbg(dev, "%s(): %d\n", __func__, data->error); return false; } - BUG_ON(!data->sg->length); + p = sgm->addr; for (i = 0; i < host->blocksize / 4; i++) *p++ = sh_mmcif_readl(host->addr, MMCIF_CE_DATA); - if (!sh_mmcif_next_block(host, p)) - return false; + sgm->consumed = host->blocksize; sh_mmcif_bitset(host, MMCIF_CE_INT_MASK, MASK_MBUFREN); + if (!sg_miter_next(sgm)) { + sg_miter_stop(sgm); + return false; + } + return true; } static void sh_mmcif_single_write(struct sh_mmcif_host *host, struct mmc_request *mrq) { + struct mmc_data *data = mrq->data; + host->blocksize = (sh_mmcif_readl(host->addr, MMCIF_CE_BLOCK_SET) & BLOCK_SIZE_MASK) + 3; + sg_miter_start(&host->sg_miter, data->sg, data->sg_len, + SG_MITER_FROM_SG); + host->wait_for = MMCIF_WAIT_FOR_WRITE; /* buf write enable */ @@ -714,20 +727,32 @@ static void sh_mmcif_single_write(struct sh_mmcif_host *host, static bool sh_mmcif_write_block(struct sh_mmcif_host *host) { + struct sg_mapping_iter *sgm = &host->sg_miter; struct device *dev = sh_mmcif_host_to_dev(host); struct mmc_data *data = host->mrq->data; - u32 *p = sg_virt(data->sg); + u32 *p; int i; if (host->sd_error) { + sg_miter_stop(sgm); data->error = sh_mmcif_error_manage(host); dev_dbg(dev, "%s(): %d\n", __func__, data->error); return false; } + if (!sg_miter_next(sgm)) { + /* This should not happen on single blocks */ + sg_miter_stop(sgm); + return false; + } + + p = sgm->addr; + for (i = 0; i < host->blocksize / 4; i++) sh_mmcif_writel(host->addr, MMCIF_CE_DATA, *p++); + sg_miter_stop(&host->sg_miter); + /* buffer write end */ sh_mmcif_bitset(host, MMCIF_CE_INT_MASK, MASK_MDTRANE); host->wait_for = MMCIF_WAIT_FOR_WRITE_END; @@ -738,6 +763,7 @@ static bool sh_mmcif_write_block(struct sh_mmcif_host *host) static void sh_mmcif_multi_write(struct sh_mmcif_host *host, struct mmc_request *mrq) { + struct sg_mapping_iter *sgm = &host->sg_miter; struct mmc_data *data = mrq->data; if (!data->sg_len || !data->sg->length) @@ -746,34 +772,46 @@ static void sh_mmcif_multi_write(struct sh_mmcif_host *host, host->blocksize = sh_mmcif_readl(host->addr, MMCIF_CE_BLOCK_SET) & BLOCK_SIZE_MASK; + sg_miter_start(sgm, data->sg, data->sg_len, + SG_MITER_FROM_SG); + + /* Advance to the first sglist entry */ + if (!sg_miter_next(sgm)) { + sg_miter_stop(sgm); + return; + } + host->wait_for = MMCIF_WAIT_FOR_MWRITE; - host->sg_idx = 0; - host->sg_blkidx = 0; - host->pio_ptr = sg_virt(data->sg); sh_mmcif_bitset(host, MMCIF_CE_INT_MASK, MASK_MBUFWEN); } static bool sh_mmcif_mwrite_block(struct sh_mmcif_host *host) { + struct sg_mapping_iter *sgm = &host->sg_miter; struct device *dev = sh_mmcif_host_to_dev(host); struct mmc_data *data = host->mrq->data; - u32 *p = host->pio_ptr; + u32 *p; int i; if (host->sd_error) { + sg_miter_stop(sgm); data->error = sh_mmcif_error_manage(host); dev_dbg(dev, "%s(): %d\n", __func__, data->error); return false; } - BUG_ON(!data->sg->length); + p = sgm->addr; for (i = 0; i < host->blocksize / 4; i++) sh_mmcif_writel(host->addr, MMCIF_CE_DATA, *p++); - if (!sh_mmcif_next_block(host, p)) + sgm->consumed = host->blocksize; + + if (!sg_miter_next(sgm)) { + sg_miter_stop(sgm); return false; + } sh_mmcif_bitset(host, MMCIF_CE_INT_MASK, MASK_MBUFWEN); diff --git a/drivers/mmc/host/tmio_mmc_core.c b/drivers/mmc/host/tmio_mmc_core.c index be7f18fd4836..93e912afd3ae 100644 --- a/drivers/mmc/host/tmio_mmc_core.c +++ b/drivers/mmc/host/tmio_mmc_core.c @@ -259,6 +259,8 @@ static void tmio_mmc_reset_work(struct work_struct *work) else mrq->cmd->error = -ETIMEDOUT; + /* No new calls yet, but disallow concurrent tmio_mmc_done_work() */ + host->mrq = ERR_PTR(-EBUSY); host->cmd = NULL; host->data = NULL; @@ -970,6 +972,7 @@ static void tmio_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) return; } + /* Disallow new mrqs and work handlers to run */ host->mrq = ERR_PTR(-EBUSY); spin_unlock_irqrestore(&host->lock, flags); @@ -1004,8 +1007,9 @@ static void tmio_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) "%s.%d: IOS interrupted: clk %u, mode %u", current->comm, task_pid_nr(current), ios->clock, ios->power_mode); - host->mrq = NULL; + /* Ready for new mrqs */ + host->mrq = NULL; host->clk_cache = ios->clock; mutex_unlock(&host->ios_lock); diff --git a/drivers/mmc/host/wbsd.c b/drivers/mmc/host/wbsd.c index 001a468bc149..f0562f712d98 100644 --- a/drivers/mmc/host/wbsd.c +++ b/drivers/mmc/host/wbsd.c @@ -1284,8 +1284,6 @@ static int wbsd_scan(struct wbsd_host *host) continue; for (j = 0; j < ARRAY_SIZE(unlock_codes); j++) { - id = 0xFFFF; - host->config = config_ports[i]; host->unlock_code = unlock_codes[j]; diff --git a/drivers/mmc/host/wmt-sdmmc.c b/drivers/mmc/host/wmt-sdmmc.c index 77d5f1d24489..860380931b6c 100644 --- a/drivers/mmc/host/wmt-sdmmc.c +++ b/drivers/mmc/host/wmt-sdmmc.c @@ -883,7 +883,6 @@ static void wmt_mci_remove(struct platform_device *pdev) { struct mmc_host *mmc; struct wmt_mci_priv *priv; - struct resource *res; u32 reg_tmp; mmc = platform_get_drvdata(pdev); @@ -911,9 +910,6 @@ static void wmt_mci_remove(struct platform_device *pdev) clk_disable_unprepare(priv->clk_sdmmc); clk_put(priv->clk_sdmmc); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - release_mem_region(res->start, resource_size(res)); - mmc_free_host(mmc); dev_info(&pdev->dev, "WMT MCI device removed\n"); diff --git a/drivers/mtd/devices/mtd_dataflash.c b/drivers/mtd/devices/mtd_dataflash.c index 0c1b93303618..ec52277e3dd5 100644 --- a/drivers/mtd/devices/mtd_dataflash.c +++ b/drivers/mtd/devices/mtd_dataflash.c @@ -638,7 +638,7 @@ static int add_dataflash_otp(struct spi_device *spi, char *name, int nr_pages, /* name must be usable with cmdlinepart */ sprintf(priv->name, "spi%d.%d-%s", - spi->master->bus_num, spi_get_chipselect(spi, 0), + spi->controller->bus_num, spi_get_chipselect(spi, 0), name); device = &priv->mtd; diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c index dea9d2907666..b08356060fb4 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_thermal.c @@ -60,7 +60,7 @@ int cxgb4_thermal_init(struct adapter *adap) snprintf(ch_tz_name, sizeof(ch_tz_name), "cxgb4_%s", adap->name); ch_thermal->tzdev = thermal_zone_device_register_with_trips(ch_tz_name, &trip, num_trip, - 0, adap, + adap, &cxgb4_thermal_ops, NULL, 0, 0); if (IS_ERR(ch_thermal->tzdev)) { diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c index f1b48d6615f6..5c511e1a8efa 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c @@ -44,16 +44,19 @@ static const struct thermal_trip default_thermal_trips[] = { .type = THERMAL_TRIP_ACTIVE, .temperature = MLXSW_THERMAL_ASIC_TEMP_NORM, .hysteresis = MLXSW_THERMAL_HYSTERESIS_TEMP, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }, { /* In range - 40-100% PWM */ .type = THERMAL_TRIP_ACTIVE, .temperature = MLXSW_THERMAL_ASIC_TEMP_HIGH, .hysteresis = MLXSW_THERMAL_HYSTERESIS_TEMP, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }, { /* Warning */ .type = THERMAL_TRIP_HOT, .temperature = MLXSW_THERMAL_ASIC_TEMP_HOT, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }, }; @@ -62,16 +65,19 @@ static const struct thermal_trip default_thermal_module_trips[] = { .type = THERMAL_TRIP_ACTIVE, .temperature = MLXSW_THERMAL_MODULE_TEMP_NORM, .hysteresis = MLXSW_THERMAL_HYSTERESIS_TEMP, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }, { /* In range - 40-100% PWM */ .type = THERMAL_TRIP_ACTIVE, .temperature = MLXSW_THERMAL_MODULE_TEMP_HIGH, .hysteresis = MLXSW_THERMAL_HYSTERESIS_TEMP, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }, { /* Warning */ .type = THERMAL_TRIP_HOT, .temperature = MLXSW_THERMAL_MODULE_TEMP_HOT, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, }, }; @@ -92,9 +98,6 @@ static const struct mlxsw_cooling_states default_cooling_states[] = { #define MLXSW_THERMAL_NUM_TRIPS ARRAY_SIZE(default_thermal_trips) -/* Make sure all trips are writable */ -#define MLXSW_THERMAL_TRIP_MASK (BIT(MLXSW_THERMAL_NUM_TRIPS) - 1) - struct mlxsw_thermal; struct mlxsw_thermal_module { @@ -420,7 +423,6 @@ mlxsw_thermal_module_tz_init(struct mlxsw_thermal_module *module_tz) module_tz->tzdev = thermal_zone_device_register_with_trips(tz_name, module_tz->trips, MLXSW_THERMAL_NUM_TRIPS, - MLXSW_THERMAL_TRIP_MASK, module_tz, &mlxsw_thermal_module_ops, &mlxsw_thermal_params, @@ -548,7 +550,6 @@ mlxsw_thermal_gearbox_tz_init(struct mlxsw_thermal_module *gearbox_tz) gearbox_tz->tzdev = thermal_zone_device_register_with_trips(tz_name, gearbox_tz->trips, MLXSW_THERMAL_NUM_TRIPS, - MLXSW_THERMAL_TRIP_MASK, gearbox_tz, &mlxsw_thermal_gearbox_ops, &mlxsw_thermal_params, 0, @@ -773,7 +774,6 @@ int mlxsw_thermal_init(struct mlxsw_core *core, thermal->tzdev = thermal_zone_device_register_with_trips("mlxsw", thermal->trips, MLXSW_THERMAL_NUM_TRIPS, - MLXSW_THERMAL_TRIP_MASK, thermal, &mlxsw_thermal_ops, &mlxsw_thermal_params, 0, diff --git a/drivers/net/ethernet/micrel/ks8851_spi.c b/drivers/net/ethernet/micrel/ks8851_spi.c index 54f2eac11a63..2f803377c9f9 100644 --- a/drivers/net/ethernet/micrel/ks8851_spi.c +++ b/drivers/net/ethernet/micrel/ks8851_spi.c @@ -156,7 +156,7 @@ static void ks8851_rdreg(struct ks8851_net *ks, unsigned int op, txb[0] = cpu_to_le16(op | KS_SPIOP_RD); - if (kss->spidev->master->flags & SPI_CONTROLLER_HALF_DUPLEX) { + if (kss->spidev->controller->flags & SPI_CONTROLLER_HALF_DUPLEX) { msg = &kss->spi_msg2; xfer = kss->spi_xfer2; @@ -180,7 +180,7 @@ static void ks8851_rdreg(struct ks8851_net *ks, unsigned int op, ret = spi_sync(kss->spidev, msg); if (ret < 0) netdev_err(ks->netdev, "read: spi_sync() failed\n"); - else if (kss->spidev->master->flags & SPI_CONTROLLER_HALF_DUPLEX) + else if (kss->spidev->controller->flags & SPI_CONTROLLER_HALF_DUPLEX) memcpy(rxb, trx, rxl); else memcpy(rxb, trx + 2, rxl); diff --git a/drivers/net/ethernet/vertexcom/mse102x.c b/drivers/net/ethernet/vertexcom/mse102x.c index aeed2a093e34..edd8b59680e5 100644 --- a/drivers/net/ethernet/vertexcom/mse102x.c +++ b/drivers/net/ethernet/vertexcom/mse102x.c @@ -664,7 +664,7 @@ static int mse102x_probe_spi(struct spi_device *spi) spi->bits_per_word = 8; spi->mode |= SPI_MODE_3; /* enforce minimum speed to ensure device functionality */ - spi->master->min_speed_hz = MIN_FREQ_HZ; + spi->controller->min_speed_hz = MIN_FREQ_HZ; if (!spi->max_speed_hz) spi->max_speed_hz = MAX_FREQ_HZ; diff --git a/drivers/net/ieee802154/ca8210.c b/drivers/net/ieee802154/ca8210.c index f102f26cb0e3..e685a7f946f0 100644 --- a/drivers/net/ieee802154/ca8210.c +++ b/drivers/net/ieee802154/ca8210.c @@ -2950,7 +2950,7 @@ static int ca8210_test_interface_init(struct ca8210_priv *priv) node_name, sizeof(node_name), "ca8210@%d_%d", - priv->spi->master->bus_num, + priv->spi->controller->bus_num, spi_get_chipselect(priv->spi, 0) ); diff --git a/drivers/net/ipa/ipa_smp2p.c b/drivers/net/ipa/ipa_smp2p.c index aeccce9fab72..2f917582c423 100644 --- a/drivers/net/ipa/ipa_smp2p.c +++ b/drivers/net/ipa/ipa_smp2p.c @@ -90,7 +90,7 @@ static void ipa_smp2p_notify(struct ipa_smp2p *smp2p) if (smp2p->notified) return; - smp2p->power_on = pm_runtime_get_if_active(smp2p->ipa->dev, true) > 0; + smp2p->power_on = pm_runtime_get_if_active(smp2p->ipa->dev) > 0; /* Signal whether the IPA power is enabled */ mask = BIT(smp2p->enabled_bit); diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h index a10b48947bca..44571114fb15 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h +++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h @@ -543,12 +543,10 @@ struct iwl_mvm_tt_mgmt { /** * struct iwl_mvm_thermal_device - thermal zone related data * @trips: temperature thresholds for report - * @fw_trips_index: keep indexes to original array - temp_trips * @tzone: thermal zone device data */ struct iwl_mvm_thermal_device { struct thermal_trip trips[IWL_MAX_DTS_TRIPS]; - u8 fw_trips_index[IWL_MAX_DTS_TRIPS]; struct thermal_zone_device *tzone; }; diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c index dee9c367dcd3..61a4638d1be2 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c @@ -555,6 +555,22 @@ static int compare_temps(const void *a, const void *b) return ((s16)le16_to_cpu(*(__le16 *)a) - (s16)le16_to_cpu(*(__le16 *)b)); } + +struct iwl_trip_walk_data { + __le16 *thresholds; + int count; +}; + +static int iwl_trip_temp_cb(struct thermal_trip *trip, void *arg) +{ + struct iwl_trip_walk_data *twd = arg; + + if (trip->temperature == THERMAL_TEMP_INVALID) + return 0; + + twd->thresholds[twd->count++] = cpu_to_le16((s16)(trip->temperature / 1000)); + return 0; +} #endif int iwl_mvm_send_temp_report_ths_cmd(struct iwl_mvm *mvm) @@ -562,42 +578,25 @@ int iwl_mvm_send_temp_report_ths_cmd(struct iwl_mvm *mvm) struct temp_report_ths_cmd cmd = {0}; int ret; #ifdef CONFIG_THERMAL - int i, j, idx = 0; + struct iwl_trip_walk_data twd = { .thresholds = cmd.thresholds, .count = 0 }; lockdep_assert_held(&mvm->mutex); if (!mvm->tz_device.tzone) goto send; - /* The driver holds array of temperature trips that are unsorted - * and uncompressed, the FW should get it compressed and sorted + /* + * The thermal core holds an array of temperature trips that are + * unsorted and uncompressed, the FW should get it compressed and + * sorted. */ /* compress trips to cmd array, remove uninitialized values*/ - for (i = 0; i < IWL_MAX_DTS_TRIPS; i++) { - if (mvm->tz_device.trips[i].temperature != INT_MIN) { - cmd.thresholds[idx++] = - cpu_to_le16((s16)(mvm->tz_device.trips[i].temperature / 1000)); - } - } - cmd.num_temps = cpu_to_le32(idx); + for_each_thermal_trip(mvm->tz_device.tzone, iwl_trip_temp_cb, &twd); - if (!idx) - goto send; - - /*sort cmd array*/ - sort(cmd.thresholds, idx, sizeof(s16), compare_temps, NULL); - - /* we should save the indexes of trips because we sort - * and compress the orginal array - */ - for (i = 0; i < idx; i++) { - for (j = 0; j < IWL_MAX_DTS_TRIPS; j++) { - if ((int)(le16_to_cpu(cmd.thresholds[i]) * 1000) == - mvm->tz_device.trips[j].temperature) - mvm->tz_device.fw_trips_index[i] = j; - } - } + cmd.num_temps = cpu_to_le32(twd.count); + if (twd.count) + sort(cmd.thresholds, twd.count, sizeof(s16), compare_temps, NULL); send: #endif @@ -668,9 +667,6 @@ static struct thermal_zone_device_ops tzone_ops = { .set_trip_temp = iwl_mvm_tzone_set_trip_temp, }; -/* make all trips writable */ -#define IWL_WRITABLE_TRIPS_MSK (BIT(IWL_MAX_DTS_TRIPS) - 1) - static void iwl_mvm_thermal_zone_register(struct iwl_mvm *mvm) { int i, ret; @@ -686,10 +682,18 @@ static void iwl_mvm_thermal_zone_register(struct iwl_mvm *mvm) BUILD_BUG_ON(ARRAY_SIZE(name) >= THERMAL_NAME_LENGTH); sprintf(name, "iwlwifi_%u", atomic_inc_return(&counter) & 0xFF); + /* + * 0 is a valid temperature, + * so initialize the array with S16_MIN which invalid temperature + */ + for (i = 0 ; i < IWL_MAX_DTS_TRIPS; i++) { + mvm->tz_device.trips[i].temperature = THERMAL_TEMP_INVALID; + mvm->tz_device.trips[i].type = THERMAL_TRIP_PASSIVE; + mvm->tz_device.trips[i].flags = THERMAL_TRIP_FLAG_RW_TEMP; + } mvm->tz_device.tzone = thermal_zone_device_register_with_trips(name, mvm->tz_device.trips, IWL_MAX_DTS_TRIPS, - IWL_WRITABLE_TRIPS_MSK, mvm, &tzone_ops, NULL, 0, 0); if (IS_ERR(mvm->tz_device.tzone)) { @@ -704,15 +708,6 @@ static void iwl_mvm_thermal_zone_register(struct iwl_mvm *mvm) if (ret) { IWL_DEBUG_TEMP(mvm, "Failed to enable thermal zone\n"); thermal_zone_device_unregister(mvm->tz_device.tzone); - return; - } - - /* 0 is a valid temperature, - * so initialize the array with S16_MIN which invalid temperature - */ - for (i = 0 ; i < IWL_MAX_DTS_TRIPS; i++) { - mvm->tz_device.trips[i].temperature = INT_MIN; - mvm->tz_device.trips[i].type = THERMAL_TRIP_PASSIVE; } } diff --git a/drivers/net/wireless/marvell/libertas/if_spi.c b/drivers/net/wireless/marvell/libertas/if_spi.c index 8690b0114e23..b722a6587fd3 100644 --- a/drivers/net/wireless/marvell/libertas/if_spi.c +++ b/drivers/net/wireless/marvell/libertas/if_spi.c @@ -1052,7 +1052,7 @@ static int if_spi_init_card(struct if_spi_card *card) "attached to SPI bus_num %d, chip_select %d. " "spi->max_speed_hz=%d\n", card->card_id, card->card_rev, - card->spi->master->bus_num, + card->spi->controller->bus_num, spi_get_chipselect(card->spi, 0), card->spi->max_speed_hz); err = if_spi_prog_helper_firmware(card, helper); diff --git a/drivers/opp/core.c b/drivers/opp/core.c index c4e0432ae42a..e233734b7220 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -2065,6 +2065,7 @@ int _opp_add_v1(struct opp_table *opp_table, struct device *dev, /* populate the opp table */ new_opp->rates[0] = data->freq; new_opp->level = data->level; + new_opp->turbo = data->turbo; tol = u_volt * opp_table->voltage_tolerance_v1 / 100; new_opp->supplies[0].u_volt = u_volt; new_opp->supplies[0].u_volt_min = u_volt - tol; diff --git a/drivers/opp/debugfs.c b/drivers/opp/debugfs.c index ec030b19164a..105de7c3274a 100644 --- a/drivers/opp/debugfs.c +++ b/drivers/opp/debugfs.c @@ -37,10 +37,12 @@ static ssize_t bw_name_read(struct file *fp, char __user *userbuf, size_t count, loff_t *ppos) { struct icc_path *path = fp->private_data; + const char *name = icc_get_name(path); char buf[64]; - int i; + int i = 0; - i = scnprintf(buf, sizeof(buf), "%.62s\n", icc_get_name(path)); + if (name) + i = scnprintf(buf, sizeof(buf), "%.62s\n", name); return simple_read_from_buffer(userbuf, count, ppos, buf, i); } @@ -56,11 +58,11 @@ static void opp_debug_create_bw(struct dev_pm_opp *opp, struct dentry *pdentry) { struct dentry *d; - char name[20]; + char name[] = "icc-path-XXXXXXXXXXX"; /* Integers can take 11 chars max */ int i; for (i = 0; i < opp_table->path_count; i++) { - snprintf(name, sizeof(name), "icc-path-%.1d", i); + snprintf(name, sizeof(name), "icc-path-%d", i); /* Create per-path directory */ d = debugfs_create_dir(name, pdentry); @@ -78,7 +80,7 @@ static void opp_debug_create_clks(struct dev_pm_opp *opp, struct opp_table *opp_table, struct dentry *pdentry) { - char name[12]; + char name[] = "rate_hz_XXXXXXXXXXX"; /* Integers can take 11 chars max */ int i; if (opp_table->clk_count == 1) { @@ -100,7 +102,7 @@ static void opp_debug_create_supplies(struct dev_pm_opp *opp, int i; for (i = 0; i < opp_table->regulator_count; i++) { - char name[15]; + char name[] = "supply-XXXXXXXXXXX"; /* Integers can take 11 chars max */ snprintf(name, sizeof(name), "supply-%d", i); diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index ccee56615f78..1e33f0e2d945 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -2532,7 +2532,7 @@ static void pci_pme_list_scan(struct work_struct *work) * course of the call. */ if (bdev) { - bref = pm_runtime_get_if_active(bdev, true); + bref = pm_runtime_get_if_active(bdev); if (!bref) continue; diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index e9750b1b19ba..bfc56f7bee1c 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -368,11 +368,6 @@ static inline int pci_dev_set_disconnected(struct pci_dev *dev, void *unused) return 0; } -static inline bool pci_dev_is_disconnected(const struct pci_dev *dev) -{ - return dev->error_state == pci_channel_io_perm_failure; -} - /* pci_dev priv_flags */ #define PCI_DEV_ADDED 0 #define PCI_DPC_RECOVERED 1 diff --git a/drivers/pinctrl/intel/pinctrl-intel.c b/drivers/pinctrl/intel/pinctrl-intel.c index d6f29e6faab7..89bd7ce6711a 100644 --- a/drivers/pinctrl/intel/pinctrl-intel.c +++ b/drivers/pinctrl/intel/pinctrl-intel.c @@ -1492,7 +1492,7 @@ static int intel_pinctrl_probe_pwm(struct intel_pinctrl *pctrl, .base_unit_bits = 22, .bypass = true, }; - struct pwm_lpss_chip *pwm; + struct pwm_chip *chip; if (!(community->features & PINCTRL_FEATURE_PWM)) return 0; @@ -1500,8 +1500,8 @@ static int intel_pinctrl_probe_pwm(struct intel_pinctrl *pctrl, if (!IS_REACHABLE(CONFIG_PWM_LPSS)) return 0; - pwm = devm_pwm_lpss_probe(pctrl->dev, community->regs + PWMC, &info); - return PTR_ERR_OR_ZERO(pwm); + chip = devm_pwm_lpss_probe(pctrl->dev, community->regs + PWMC, &info); + return PTR_ERR_OR_ZERO(chip); } int intel_pinctrl_probe(struct platform_device *pdev, diff --git a/drivers/platform/chrome/cros_ec_spi.c b/drivers/platform/chrome/cros_ec_spi.c index 3e88cc92e819..86a3d32a7763 100644 --- a/drivers/platform/chrome/cros_ec_spi.c +++ b/drivers/platform/chrome/cros_ec_spi.c @@ -409,7 +409,7 @@ static int do_cros_ec_pkt_xfer_spi(struct cros_ec_device *ec_dev, if (!rx_buf) return -ENOMEM; - spi_bus_lock(ec_spi->spi->master); + spi_bus_lock(ec_spi->spi->controller); /* * Leave a gap between CS assertion and clocking of data to allow the @@ -469,7 +469,7 @@ static int do_cros_ec_pkt_xfer_spi(struct cros_ec_device *ec_dev, final_ret = terminate_request(ec_dev); - spi_bus_unlock(ec_spi->spi->master); + spi_bus_unlock(ec_spi->spi->controller); if (!ret) ret = final_ret; @@ -554,7 +554,7 @@ static int do_cros_ec_cmd_xfer_spi(struct cros_ec_device *ec_dev, if (!rx_buf) return -ENOMEM; - spi_bus_lock(ec_spi->spi->master); + spi_bus_lock(ec_spi->spi->controller); /* Transmit phase - send our message */ debug_packet(ec_dev->dev, "out", ec_dev->dout, len); @@ -590,7 +590,7 @@ static int do_cros_ec_cmd_xfer_spi(struct cros_ec_device *ec_dev, final_ret = terminate_request(ec_dev); - spi_bus_unlock(ec_spi->spi->master); + spi_bus_unlock(ec_spi->spi->controller); if (!ret) ret = final_ret; diff --git a/drivers/platform/x86/acerhdf.c b/drivers/platform/x86/acerhdf.c index 74bcb3d13104..018c48429616 100644 --- a/drivers/platform/x86/acerhdf.c +++ b/drivers/platform/x86/acerhdf.c @@ -678,7 +678,7 @@ static int __init acerhdf_register_thermal(void) return -EINVAL; thz_dev = thermal_zone_device_register_with_trips("acerhdf", trips, ARRAY_SIZE(trips), - 0, NULL, &acerhdf_dev_ops, + NULL, &acerhdf_dev_ops, &acerhdf_zone_params, 0, (kernelmode) ? interval*1000 : 0); if (IS_ERR(thz_dev)) diff --git a/drivers/pmdomain/core.c b/drivers/pmdomain/core.c index 18e232b5ed53..4215ffd9b11c 100644 --- a/drivers/pmdomain/core.c +++ b/drivers/pmdomain/core.c @@ -310,73 +310,103 @@ static int genpd_xlate_performance_state(struct generic_pm_domain *genpd, pstate); } +static int _genpd_set_performance_state(struct generic_pm_domain *genpd, + unsigned int state, int depth); + +static void _genpd_rollback_parent_state(struct gpd_link *link, int depth) +{ + struct generic_pm_domain *parent = link->parent; + int parent_state; + + genpd_lock_nested(parent, depth + 1); + + parent_state = link->prev_performance_state; + link->performance_state = parent_state; + + parent_state = _genpd_reeval_performance_state(parent, parent_state); + if (_genpd_set_performance_state(parent, parent_state, depth + 1)) { + pr_err("%s: Failed to roll back to %d performance state\n", + parent->name, parent_state); + } + + genpd_unlock(parent); +} + +static int _genpd_set_parent_state(struct generic_pm_domain *genpd, + struct gpd_link *link, + unsigned int state, int depth) +{ + struct generic_pm_domain *parent = link->parent; + int parent_state, ret; + + /* Find parent's performance state */ + ret = genpd_xlate_performance_state(genpd, parent, state); + if (unlikely(ret < 0)) + return ret; + + parent_state = ret; + + genpd_lock_nested(parent, depth + 1); + + link->prev_performance_state = link->performance_state; + link->performance_state = parent_state; + + parent_state = _genpd_reeval_performance_state(parent, parent_state); + ret = _genpd_set_performance_state(parent, parent_state, depth + 1); + if (ret) + link->performance_state = link->prev_performance_state; + + genpd_unlock(parent); + + return ret; +} + static int _genpd_set_performance_state(struct generic_pm_domain *genpd, unsigned int state, int depth) { - struct generic_pm_domain *parent; - struct gpd_link *link; - int parent_state, ret; + struct gpd_link *link = NULL; + int ret; if (state == genpd->performance_state) return 0; - /* Propagate to parents of genpd */ - list_for_each_entry(link, &genpd->child_links, child_node) { - parent = link->parent; - - /* Find parent's performance state */ - ret = genpd_xlate_performance_state(genpd, parent, state); - if (unlikely(ret < 0)) - goto err; - - parent_state = ret; - - genpd_lock_nested(parent, depth + 1); - - link->prev_performance_state = link->performance_state; - link->performance_state = parent_state; - parent_state = _genpd_reeval_performance_state(parent, - parent_state); - ret = _genpd_set_performance_state(parent, parent_state, depth + 1); - if (ret) - link->performance_state = link->prev_performance_state; - - genpd_unlock(parent); - - if (ret) - goto err; + /* When scaling up, propagate to parents first in normal order */ + if (state > genpd->performance_state) { + list_for_each_entry(link, &genpd->child_links, child_node) { + ret = _genpd_set_parent_state(genpd, link, state, depth); + if (ret) + goto rollback_parents_up; + } } if (genpd->set_performance_state) { ret = genpd->set_performance_state(genpd, state); - if (ret) - goto err; + if (ret) { + if (link) + goto rollback_parents_up; + return ret; + } + } + + /* When scaling down, propagate to parents last in reverse order */ + if (state < genpd->performance_state) { + list_for_each_entry_reverse(link, &genpd->child_links, child_node) { + ret = _genpd_set_parent_state(genpd, link, state, depth); + if (ret) + goto rollback_parents_down; + } } genpd->performance_state = state; return 0; -err: - /* Encountered an error, lets rollback */ - list_for_each_entry_continue_reverse(link, &genpd->child_links, - child_node) { - parent = link->parent; - - genpd_lock_nested(parent, depth + 1); - - parent_state = link->prev_performance_state; - link->performance_state = parent_state; - - parent_state = _genpd_reeval_performance_state(parent, - parent_state); - if (_genpd_set_performance_state(parent, parent_state, depth + 1)) { - pr_err("%s: Failed to roll back to %d performance state\n", - parent->name, parent_state); - } - - genpd_unlock(parent); - } - +rollback_parents_up: + list_for_each_entry_continue_reverse(link, &genpd->child_links, child_node) + _genpd_rollback_parent_state(link, depth); + return ret; +rollback_parents_down: + list_for_each_entry_continue(link, &genpd->child_links, child_node) + _genpd_rollback_parent_state(link, depth); return ret; } @@ -1100,6 +1130,7 @@ static int __init genpd_power_off_unused(void) return 0; } + pr_info("genpd: Disabling unused power domains\n"); mutex_lock(&gpd_list_lock); list_for_each_entry(genpd, &gpd_list, gpd_list_node) @@ -2235,7 +2266,7 @@ static DEFINE_MUTEX(of_genpd_mutex); * to be a valid pointer to struct generic_pm_domain. */ static struct generic_pm_domain *genpd_xlate_simple( - struct of_phandle_args *genpdspec, + const struct of_phandle_args *genpdspec, void *data) { return data; @@ -2252,7 +2283,7 @@ static struct generic_pm_domain *genpd_xlate_simple( * the genpd_onecell_data struct when registering the provider. */ static struct generic_pm_domain *genpd_xlate_onecell( - struct of_phandle_args *genpdspec, + const struct of_phandle_args *genpdspec, void *data) { struct genpd_onecell_data *genpd_data = data; @@ -2495,7 +2526,7 @@ EXPORT_SYMBOL_GPL(of_genpd_del_provider); * on failure. */ static struct generic_pm_domain *genpd_get_from_provider( - struct of_phandle_args *genpdspec) + const struct of_phandle_args *genpdspec) { struct generic_pm_domain *genpd = ERR_PTR(-ENOENT); struct of_genpd_provider *provider; @@ -2526,7 +2557,7 @@ static struct generic_pm_domain *genpd_get_from_provider( * Looks-up an I/O PM domain based upon phandle args provided and adds * the device to the PM domain. Returns a negative error code on failure. */ -int of_genpd_add_device(struct of_phandle_args *genpdspec, struct device *dev) +int of_genpd_add_device(const struct of_phandle_args *genpdspec, struct device *dev) { struct generic_pm_domain *genpd; int ret; @@ -2560,8 +2591,8 @@ EXPORT_SYMBOL_GPL(of_genpd_add_device); * provided and adds the subdomain to the parent PM domain. Returns a * negative error code on failure. */ -int of_genpd_add_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec) +int of_genpd_add_subdomain(const struct of_phandle_args *parent_spec, + const struct of_phandle_args *subdomain_spec) { struct generic_pm_domain *parent, *subdomain; int ret; @@ -2598,8 +2629,8 @@ EXPORT_SYMBOL_GPL(of_genpd_add_subdomain); * provided and removes the subdomain from the parent PM domain. Returns a * negative error code on failure. */ -int of_genpd_remove_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec) +int of_genpd_remove_subdomain(const struct of_phandle_args *parent_spec, + const struct of_phandle_args *subdomain_spec) { struct generic_pm_domain *parent, *subdomain; int ret; diff --git a/drivers/pmdomain/imx/imx8m-blk-ctrl.c b/drivers/pmdomain/imx/imx8m-blk-ctrl.c index 1341a707f61b..ca942d7929c2 100644 --- a/drivers/pmdomain/imx/imx8m-blk-ctrl.c +++ b/drivers/pmdomain/imx/imx8m-blk-ctrl.c @@ -258,11 +258,14 @@ static int imx8m_blk_ctrl_probe(struct platform_device *pdev) domain->power_dev = dev_pm_domain_attach_by_name(dev, data->gpc_name); - if (IS_ERR(domain->power_dev)) { - dev_err_probe(dev, PTR_ERR(domain->power_dev), + if (IS_ERR_OR_NULL(domain->power_dev)) { + if (!domain->power_dev) + ret = -ENODEV; + else + ret = PTR_ERR(domain->power_dev); + dev_err_probe(dev, ret, "failed to attach power domain \"%s\"\n", data->gpc_name); - ret = PTR_ERR(domain->power_dev); goto cleanup_pds; } diff --git a/drivers/pmdomain/imx/imx8mp-blk-ctrl.c b/drivers/pmdomain/imx/imx8mp-blk-ctrl.c index e3203eb6a022..77e889165eed 100644 --- a/drivers/pmdomain/imx/imx8mp-blk-ctrl.c +++ b/drivers/pmdomain/imx/imx8mp-blk-ctrl.c @@ -55,7 +55,7 @@ struct imx8mp_blk_ctrl_domain_data { const char *gpc_name; }; -#define DOMAIN_MAX_CLKS 2 +#define DOMAIN_MAX_CLKS 3 #define DOMAIN_MAX_PATHS 3 struct imx8mp_blk_ctrl_domain { @@ -457,8 +457,8 @@ static const struct imx8mp_blk_ctrl_domain_data imx8mp_hdmi_domain_data[] = { }, [IMX8MP_HDMIBLK_PD_LCDIF] = { .name = "hdmiblk-lcdif", - .clk_names = (const char *[]){ "axi", "apb" }, - .num_clks = 2, + .clk_names = (const char *[]){ "axi", "apb", "fdcc" }, + .num_clks = 3, .gpc_name = "lcdif", .path_names = (const char *[]){"lcdif-hdmi"}, .num_paths = 1, @@ -483,8 +483,8 @@ static const struct imx8mp_blk_ctrl_domain_data imx8mp_hdmi_domain_data[] = { }, [IMX8MP_HDMIBLK_PD_HDMI_TX] = { .name = "hdmiblk-hdmi-tx", - .clk_names = (const char *[]){ "apb", "ref_266m" }, - .num_clks = 2, + .clk_names = (const char *[]){ "apb", "ref_266m", "fdcc" }, + .num_clks = 3, .gpc_name = "hdmi-tx", }, [IMX8MP_HDMIBLK_PD_HDMI_TX_PHY] = { @@ -687,11 +687,14 @@ static int imx8mp_blk_ctrl_probe(struct platform_device *pdev) domain->power_dev = dev_pm_domain_attach_by_name(dev, data->gpc_name); - if (IS_ERR(domain->power_dev)) { - dev_err_probe(dev, PTR_ERR(domain->power_dev), + if (IS_ERR_OR_NULL(domain->power_dev)) { + if (!domain->power_dev) + ret = -ENODEV; + else + ret = PTR_ERR(domain->power_dev); + dev_err_probe(dev, ret, "failed to attach power domain %s\n", data->gpc_name); - ret = PTR_ERR(domain->power_dev); goto cleanup_pds; } diff --git a/drivers/pmdomain/imx/scu-pd.c b/drivers/pmdomain/imx/scu-pd.c index 891c1d925a9d..05841b0bf7f3 100644 --- a/drivers/pmdomain/imx/scu-pd.c +++ b/drivers/pmdomain/imx/scu-pd.c @@ -393,7 +393,7 @@ static int imx_sc_pd_power_off(struct generic_pm_domain *domain) return imx_sc_pd_power(domain, false); } -static struct generic_pm_domain *imx_scu_pd_xlate(struct of_phandle_args *spec, +static struct generic_pm_domain *imx_scu_pd_xlate(const struct of_phandle_args *spec, void *data) { struct generic_pm_domain *domain = ERR_PTR(-ENOENT); diff --git a/drivers/pmdomain/mediatek/mtk-scpsys.c b/drivers/pmdomain/mediatek/mtk-scpsys.c index b374d01fdac7..59a7a8c261ed 100644 --- a/drivers/pmdomain/mediatek/mtk-scpsys.c +++ b/drivers/pmdomain/mediatek/mtk-scpsys.c @@ -425,7 +425,6 @@ static struct scp *init_scp(struct platform_device *pdev, bool bus_prot_reg_update) { struct genpd_onecell_data *pd_data; - struct resource *res; int i, j; struct scp *scp; struct clk *clk[CLK_MAX]; @@ -441,8 +440,7 @@ static struct scp *init_scp(struct platform_device *pdev, scp->dev = &pdev->dev; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - scp->base = devm_ioremap_resource(&pdev->dev, res); + scp->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(scp->base)) return ERR_CAST(scp->base); diff --git a/drivers/pmdomain/qcom/rpmpd.c b/drivers/pmdomain/qcom/rpmpd.c index 7796d65f96e8..5e6280b4cf70 100644 --- a/drivers/pmdomain/qcom/rpmpd.c +++ b/drivers/pmdomain/qcom/rpmpd.c @@ -16,6 +16,8 @@ #define domain_to_rpmpd(domain) container_of(domain, struct rpmpd, pd) +static struct qcom_smd_rpm *rpmpd_smd_rpm; + /* Resource types: * RPMPD_X is X encoded as a little-endian, lower-case, ASCII string */ #define RPMPD_SMPA 0x61706d73 @@ -54,7 +56,6 @@ struct rpmpd { bool enabled; const int res_type; const int res_id; - struct qcom_smd_rpm *rpm; unsigned int max_state; __le32 key; bool state_synced; @@ -226,7 +227,46 @@ static struct rpmpd cx_s3a_vfl = { .key = KEY_FLOOR_LEVEL, }; +static struct rpmpd cx_s2b_corner_ao; +static struct rpmpd cx_s2b_corner = { + .pd = { .name = "cx", }, + .peer = &cx_s2b_corner_ao, + .res_type = RPMPD_SMPB, + .res_id = 2, + .key = KEY_CORNER, +}; + +static struct rpmpd cx_s2b_corner_ao = { + .pd = { .name = "cx_ao", }, + .peer = &cx_s2b_corner, + .active_only = true, + .res_type = RPMPD_SMPB, + .res_id = 2, + .key = KEY_CORNER, +}; + +static struct rpmpd cx_s2b_vfc = { + .pd = { .name = "cx_vfc", }, + .res_type = RPMPD_SMPB, + .res_id = 2, + .key = KEY_FLOOR_CORNER, +}; + /* G(F)X */ +static struct rpmpd gfx_s7a_corner = { + .pd = { .name = "gfx", }, + .res_type = RPMPD_SMPA, + .res_id = 7, + .key = KEY_CORNER, +}; + +static struct rpmpd gfx_s7a_vfc = { + .pd = { .name = "gfx_vfc", }, + .res_type = RPMPD_SMPA, + .res_id = 7, + .key = KEY_FLOOR_CORNER, +}; + static struct rpmpd gfx_s2b_corner = { .pd = { .name = "gfx", }, .res_type = RPMPD_SMPB, @@ -241,6 +281,20 @@ static struct rpmpd gfx_s2b_vfc = { .key = KEY_FLOOR_CORNER, }; +static struct rpmpd gfx_s4b_corner = { + .pd = { .name = "gfx", }, + .res_type = RPMPD_SMPB, + .res_id = 4, + .key = KEY_CORNER, +}; + +static struct rpmpd gfx_s4b_vfc = { + .pd = { .name = "gfx_vfc", }, + .res_type = RPMPD_SMPB, + .res_id = 4, + .key = KEY_FLOOR_CORNER, +}; + static struct rpmpd mx_rwmx0_lvl; static struct rpmpd gx_rwgx0_lvl_ao; static struct rpmpd gx_rwgx0_lvl = { @@ -663,6 +717,34 @@ static const struct rpmpd_desc msm8953_desc = { .max_state = RPM_SMD_LEVEL_TURBO, }; +static struct rpmpd *msm8974_rpmpds[] = { + [MSM8974_VDDCX] = &cx_s2b_corner, + [MSM8974_VDDCX_AO] = &cx_s2b_corner_ao, + [MSM8974_VDDCX_VFC] = &cx_s2b_vfc, + [MSM8974_VDDGFX] = &gfx_s4b_corner, + [MSM8974_VDDGFX_VFC] = &gfx_s4b_vfc, +}; + +static const struct rpmpd_desc msm8974_desc = { + .rpmpds = msm8974_rpmpds, + .num_pds = ARRAY_SIZE(msm8974_rpmpds), + .max_state = MAX_CORNER_RPMPD_STATE, +}; + +static struct rpmpd *msm8974pro_pma8084_rpmpds[] = { + [MSM8974_VDDCX] = &cx_s2a_corner, + [MSM8974_VDDCX_AO] = &cx_s2a_corner_ao, + [MSM8974_VDDCX_VFC] = &cx_s2a_vfc, + [MSM8974_VDDGFX] = &gfx_s7a_corner, + [MSM8974_VDDGFX_VFC] = &gfx_s7a_vfc, +}; + +static const struct rpmpd_desc msm8974pro_pma8084_desc = { + .rpmpds = msm8974pro_pma8084_rpmpds, + .num_pds = ARRAY_SIZE(msm8974pro_pma8084_rpmpds), + .max_state = MAX_CORNER_RPMPD_STATE, +}; + static struct rpmpd *msm8976_rpmpds[] = { [MSM8976_VDDCX] = &cx_s2a_lvl, [MSM8976_VDDCX_AO] = &cx_s2a_lvl_ao, @@ -856,6 +938,8 @@ static const struct of_device_id rpmpd_match_table[] = { { .compatible = "qcom,msm8917-rpmpd", .data = &msm8917_desc }, { .compatible = "qcom,msm8939-rpmpd", .data = &msm8939_desc }, { .compatible = "qcom,msm8953-rpmpd", .data = &msm8953_desc }, + { .compatible = "qcom,msm8974-rpmpd", .data = &msm8974_desc }, + { .compatible = "qcom,msm8974pro-pma8084-rpmpd", .data = &msm8974pro_pma8084_desc }, { .compatible = "qcom,msm8976-rpmpd", .data = &msm8976_desc }, { .compatible = "qcom,msm8994-rpmpd", .data = &msm8994_desc }, { .compatible = "qcom,msm8996-rpmpd", .data = &msm8996_desc }, @@ -879,7 +963,7 @@ static int rpmpd_send_enable(struct rpmpd *pd, bool enable) .value = cpu_to_le32(enable), }; - return qcom_rpm_smd_write(pd->rpm, QCOM_SMD_RPM_ACTIVE_STATE, + return qcom_rpm_smd_write(rpmpd_smd_rpm, QCOM_SMD_RPM_ACTIVE_STATE, pd->res_type, pd->res_id, &req, sizeof(req)); } @@ -891,7 +975,7 @@ static int rpmpd_send_corner(struct rpmpd *pd, int state, unsigned int corner) .value = cpu_to_le32(corner), }; - return qcom_rpm_smd_write(pd->rpm, state, pd->res_type, pd->res_id, + return qcom_rpm_smd_write(rpmpd_smd_rpm, state, pd->res_type, pd->res_id, &req, sizeof(req)); }; @@ -1004,12 +1088,11 @@ static int rpmpd_probe(struct platform_device *pdev) int i; size_t num; struct genpd_onecell_data *data; - struct qcom_smd_rpm *rpm; struct rpmpd **rpmpds; const struct rpmpd_desc *desc; - rpm = dev_get_drvdata(pdev->dev.parent); - if (!rpm) { + rpmpd_smd_rpm = dev_get_drvdata(pdev->dev.parent); + if (!rpmpd_smd_rpm) { dev_err(&pdev->dev, "Unable to retrieve handle to RPM\n"); return -ENODEV; } @@ -1039,7 +1122,6 @@ static int rpmpd_probe(struct platform_device *pdev) continue; } - rpmpds[i]->rpm = rpm; rpmpds[i]->max_state = desc->max_state; rpmpds[i]->pd.power_off = rpmpd_power_off; rpmpds[i]->pd.power_on = rpmpd_power_on; diff --git a/drivers/pmdomain/renesas/Kconfig b/drivers/pmdomain/renesas/Kconfig index 80bf2cf8b60e..54acb4b1ec7c 100644 --- a/drivers/pmdomain/renesas/Kconfig +++ b/drivers/pmdomain/renesas/Kconfig @@ -71,6 +71,10 @@ config SYSC_R8A779G0 bool "System Controller support for R-Car V4H" if COMPILE_TEST select SYSC_RCAR_GEN4 +config SYSC_R8A779H0 + bool "System Controller support for R-Car V4M" if COMPILE_TEST + select SYSC_RCAR_GEN4 + config SYSC_RMOBILE bool "System Controller support for R-Mobile" if COMPILE_TEST diff --git a/drivers/pmdomain/renesas/Makefile b/drivers/pmdomain/renesas/Makefile index e306e396fc8c..89180f19c23b 100644 --- a/drivers/pmdomain/renesas/Makefile +++ b/drivers/pmdomain/renesas/Makefile @@ -24,6 +24,7 @@ obj-$(CONFIG_SYSC_R8A77995) += r8a77995-sysc.o obj-$(CONFIG_SYSC_R8A779A0) += r8a779a0-sysc.o obj-$(CONFIG_SYSC_R8A779F0) += r8a779f0-sysc.o obj-$(CONFIG_SYSC_R8A779G0) += r8a779g0-sysc.o +obj-$(CONFIG_SYSC_R8A779H0) += r8a779h0-sysc.o # Family obj-$(CONFIG_SYSC_RCAR) += rcar-sysc.o obj-$(CONFIG_SYSC_RCAR_GEN4) += rcar-gen4-sysc.o diff --git a/drivers/pmdomain/renesas/r8a779a0-sysc.c b/drivers/pmdomain/renesas/r8a779a0-sysc.c index 04f1bc322ae7..54cdf250f7c2 100644 --- a/drivers/pmdomain/renesas/r8a779a0-sysc.c +++ b/drivers/pmdomain/renesas/r8a779a0-sysc.c @@ -5,19 +5,7 @@ * Copyright (C) 2020 Renesas Electronics Corp. */ -#include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include #include diff --git a/drivers/pmdomain/renesas/r8a779f0-sysc.c b/drivers/pmdomain/renesas/r8a779f0-sysc.c index 5602aa6bd7ed..6ed13cd1cb24 100644 --- a/drivers/pmdomain/renesas/r8a779f0-sysc.c +++ b/drivers/pmdomain/renesas/r8a779f0-sysc.c @@ -5,19 +5,7 @@ * Copyright (C) 2021 Renesas Electronics Corp. */ -#include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include #include diff --git a/drivers/pmdomain/renesas/r8a779g0-sysc.c b/drivers/pmdomain/renesas/r8a779g0-sysc.c index b932eba1b804..249cf43af45b 100644 --- a/drivers/pmdomain/renesas/r8a779g0-sysc.c +++ b/drivers/pmdomain/renesas/r8a779g0-sysc.c @@ -5,19 +5,7 @@ * Copyright (C) 2022 Renesas Electronics Corp. */ -#include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include #include diff --git a/drivers/pmdomain/renesas/r8a779h0-sysc.c b/drivers/pmdomain/renesas/r8a779h0-sysc.c new file mode 100644 index 000000000000..e13372cb80ee --- /dev/null +++ b/drivers/pmdomain/renesas/r8a779h0-sysc.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Renesas R-Car V4M System Controller + * + * Copyright (C) 2023 Renesas Electronics Corp + */ + +#include + +#include + +#include "rcar-gen4-sysc.h" + +static struct rcar_gen4_sysc_area r8a779h0_areas[] __initdata = { + { "always-on", R8A779H0_PD_ALWAYS_ON, -1, PD_ALWAYS_ON }, + { "c4", R8A779H0_PD_C4, R8A779H0_PD_ALWAYS_ON }, + { "a2e0d0", R8A779H0_PD_A2E0D0, R8A779H0_PD_C4, PD_SCU }, + { "a1e0d0c0", R8A779H0_PD_A1E0D0C0, R8A779H0_PD_A2E0D0, PD_CPU_NOCR }, + { "a1e0d0c1", R8A779H0_PD_A1E0D0C1, R8A779H0_PD_A2E0D0, PD_CPU_NOCR }, + { "a1e0d0c2", R8A779H0_PD_A1E0D0C2, R8A779H0_PD_A2E0D0, PD_CPU_NOCR }, + { "a1e0d0c3", R8A779H0_PD_A1E0D0C3, R8A779H0_PD_A2E0D0, PD_CPU_NOCR }, + { "a3cr0", R8A779H0_PD_A3CR0, R8A779H0_PD_ALWAYS_ON, PD_CPU_NOCR }, + { "a3cr1", R8A779H0_PD_A3CR1, R8A779H0_PD_ALWAYS_ON, PD_CPU_NOCR }, + { "a3cr2", R8A779H0_PD_A3CR2, R8A779H0_PD_ALWAYS_ON, PD_CPU_NOCR }, + { "a33dga", R8A779H0_PD_A33DGA, R8A779H0_PD_C4 }, + { "a23dgb", R8A779H0_PD_A23DGB, R8A779H0_PD_A33DGA }, + { "a3vip0", R8A779H0_PD_A3VIP0, R8A779H0_PD_C4 }, + { "a3vip2", R8A779H0_PD_A3VIP2, R8A779H0_PD_C4 }, + { "a3dul", R8A779H0_PD_A3DUL, R8A779H0_PD_C4 }, + { "a3isp0", R8A779H0_PD_A3ISP0, R8A779H0_PD_C4 }, + { "a2cn0", R8A779H0_PD_A2CN0, R8A779H0_PD_C4 }, + { "a1cn0", R8A779H0_PD_A1CN0, R8A779H0_PD_A2CN0 }, + { "a1dsp0", R8A779H0_PD_A1DSP0, R8A779H0_PD_A2CN0 }, + { "a1dsp1", R8A779H0_PD_A1DSP1, R8A779H0_PD_A2CN0 }, + { "a2imp01", R8A779H0_PD_A2IMP01, R8A779H0_PD_C4 }, + { "a2psc", R8A779H0_PD_A2PSC, R8A779H0_PD_C4 }, + { "a2dma", R8A779H0_PD_A2DMA, R8A779H0_PD_C4 }, + { "a2cv0", R8A779H0_PD_A2CV0, R8A779H0_PD_C4 }, + { "a2cv1", R8A779H0_PD_A2CV1, R8A779H0_PD_C4 }, + { "a2cv2", R8A779H0_PD_A2CV2, R8A779H0_PD_C4 }, + { "a2cv3", R8A779H0_PD_A2CV3, R8A779H0_PD_C4 }, + { "a3imr0", R8A779H0_PD_A3IMR0, R8A779H0_PD_C4 }, + { "a3imr1", R8A779H0_PD_A3IMR1, R8A779H0_PD_C4 }, + { "a3imr2", R8A779H0_PD_A3IMR2, R8A779H0_PD_C4 }, + { "a3imr3", R8A779H0_PD_A3IMR3, R8A779H0_PD_C4 }, + { "a3vc", R8A779H0_PD_A3VC, R8A779H0_PD_C4 }, + { "a3pci", R8A779H0_PD_A3PCI, R8A779H0_PD_C4 }, + { "a2pciphy", R8A779H0_PD_A2PCIPHY, R8A779H0_PD_A3PCI }, +}; + +const struct rcar_gen4_sysc_info r8a779h0_sysc_info __initconst = { + .areas = r8a779h0_areas, + .num_areas = ARRAY_SIZE(r8a779h0_areas), +}; diff --git a/drivers/pmdomain/renesas/rcar-gen4-sysc.c b/drivers/pmdomain/renesas/rcar-gen4-sysc.c index 9e5e6e077abc..66409cff2083 100644 --- a/drivers/pmdomain/renesas/rcar-gen4-sysc.c +++ b/drivers/pmdomain/renesas/rcar-gen4-sysc.c @@ -50,13 +50,13 @@ #define SYSCSR_BUSY GENMASK(1, 0) /* All bit sets is not busy */ #define SYSCSR_TIMEOUT 10000 -#define SYSCSR_DELAY_US 10 +#define SYSCSR_DELAY_US 1 -#define PDRESR_RETRIES 1000 -#define PDRESR_DELAY_US 10 +#define PDRESR_RETRIES 10000 +#define PDRESR_DELAY_US 1 -#define SYSCISR_TIMEOUT 10000 -#define SYSCISR_DELAY_US 10 +#define SYSCISCR_TIMEOUT 10000 +#define SYSCISCR_DELAY_US 1 #define RCAR_GEN4_PD_ALWAYS_ON 64 #define NUM_DOMAINS_EACH_REG BITS_PER_TYPE(u32) @@ -97,7 +97,7 @@ static int clear_irq_flags(unsigned int reg_idx, unsigned int isr_mask) ret = readl_poll_timeout_atomic(rcar_gen4_sysc_base + SYSCISCR(reg_idx), val, !(val & isr_mask), - SYSCISR_DELAY_US, SYSCISR_TIMEOUT); + SYSCISCR_DELAY_US, SYSCISCR_TIMEOUT); if (ret < 0) { pr_err("\n %s : Can not clear IRQ flags in SYSCISCR", __func__); return -EIO; @@ -157,7 +157,7 @@ static int rcar_gen4_sysc_power(u8 pdr, bool on) /* Wait until the power shutoff or resume request has completed * */ ret = readl_poll_timeout_atomic(rcar_gen4_sysc_base + SYSCISCR(reg_idx), val, (val & isr_mask), - SYSCISR_DELAY_US, SYSCISR_TIMEOUT); + SYSCISCR_DELAY_US, SYSCISCR_TIMEOUT); if (ret < 0) { ret = -EIO; goto out; @@ -284,6 +284,9 @@ static const struct of_device_id rcar_gen4_sysc_matches[] __initconst = { #endif #ifdef CONFIG_SYSC_R8A779G0 { .compatible = "renesas,r8a779g0-sysc", .data = &r8a779g0_sysc_info }, +#endif +#ifdef CONFIG_SYSC_R8A779H0 + { .compatible = "renesas,r8a779h0-sysc", .data = &r8a779h0_sysc_info }, #endif { /* sentinel */ } }; diff --git a/drivers/pmdomain/renesas/rcar-gen4-sysc.h b/drivers/pmdomain/renesas/rcar-gen4-sysc.h index 388cfa8f8f9f..fdf843aa5113 100644 --- a/drivers/pmdomain/renesas/rcar-gen4-sysc.h +++ b/drivers/pmdomain/renesas/rcar-gen4-sysc.h @@ -40,5 +40,6 @@ struct rcar_gen4_sysc_info { extern const struct rcar_gen4_sysc_info r8a779a0_sysc_info; extern const struct rcar_gen4_sysc_info r8a779f0_sysc_info; extern const struct rcar_gen4_sysc_info r8a779g0_sysc_info; +extern const struct rcar_gen4_sysc_info r8a779h0_sysc_info; #endif /* __SOC_RENESAS_RCAR_GEN4_SYSC_H__ */ diff --git a/drivers/pmdomain/renesas/rcar-sysc.c b/drivers/pmdomain/renesas/rcar-sysc.c index eed47696e825..35d9aa0dfab8 100644 --- a/drivers/pmdomain/renesas/rcar-sysc.c +++ b/drivers/pmdomain/renesas/rcar-sysc.c @@ -45,10 +45,10 @@ #define PWRER_OFFS 0x14 /* Power Shutoff/Resume Error */ -#define SYSCSR_TIMEOUT 100 +#define SYSCSR_TIMEOUT 1000 #define SYSCSR_DELAY_US 1 -#define PWRER_RETRIES 100 +#define PWRER_RETRIES 1000 #define PWRER_DELAY_US 1 #define SYSCISR_TIMEOUT 1000 diff --git a/drivers/pmdomain/tegra/powergate-bpmp.c b/drivers/pmdomain/tegra/powergate-bpmp.c index 179ed895c279..b0138ca9f851 100644 --- a/drivers/pmdomain/tegra/powergate-bpmp.c +++ b/drivers/pmdomain/tegra/powergate-bpmp.c @@ -305,7 +305,7 @@ static void tegra_bpmp_remove_powergates(struct tegra_bpmp *bpmp) } static struct generic_pm_domain * -tegra_powergate_xlate(struct of_phandle_args *spec, void *data) +tegra_powergate_xlate(const struct of_phandle_args *spec, void *data) { struct generic_pm_domain *domain = ERR_PTR(-ENOENT); struct genpd_onecell_data *genpd = data; diff --git a/drivers/pmdomain/ti/omap_prm.c b/drivers/pmdomain/ti/omap_prm.c index c2feae3a634c..b8ceb3c2b81c 100644 --- a/drivers/pmdomain/ti/omap_prm.c +++ b/drivers/pmdomain/ti/omap_prm.c @@ -695,6 +695,8 @@ static int omap_prm_domain_init(struct device *dev, struct omap_prm *prm) data = prm->data; name = devm_kasprintf(dev, GFP_KERNEL, "prm_%s", data->name); + if (!name) + return -ENOMEM; prmd->dev = dev; prmd->prm = prm; diff --git a/drivers/pmdomain/ti/ti_sci_pm_domains.c b/drivers/pmdomain/ti/ti_sci_pm_domains.c index c091d569ecd5..9dddf227a3a6 100644 --- a/drivers/pmdomain/ti/ti_sci_pm_domains.c +++ b/drivers/pmdomain/ti/ti_sci_pm_domains.c @@ -85,7 +85,7 @@ static int ti_sci_pd_power_on(struct generic_pm_domain *domain) * @data: genpd core data for all the powerdomains on the device */ static struct generic_pm_domain *ti_sci_pd_xlate( - struct of_phandle_args *genpdspec, + const struct of_phandle_args *genpdspec, void *data) { struct genpd_onecell_data *genpd_data = data; diff --git a/drivers/pmdomain/xilinx/zynqmp-pm-domains.c b/drivers/pmdomain/xilinx/zynqmp-pm-domains.c index 6fd514286d82..0b5831e5ba1b 100644 --- a/drivers/pmdomain/xilinx/zynqmp-pm-domains.c +++ b/drivers/pmdomain/xilinx/zynqmp-pm-domains.c @@ -210,7 +210,7 @@ static void zynqmp_gpd_detach_dev(struct generic_pm_domain *domain, } static struct generic_pm_domain *zynqmp_gpd_xlate - (struct of_phandle_args *genpdspec, void *data) + (const struct of_phandle_args *genpdspec, void *data) { struct genpd_onecell_data *genpd_data = data; unsigned int i, idx = genpdspec->args[0]; diff --git a/drivers/powercap/dtpm.c b/drivers/powercap/dtpm.c index ce920f17f45f..f390665743c4 100644 --- a/drivers/powercap/dtpm.c +++ b/drivers/powercap/dtpm.c @@ -522,7 +522,7 @@ static int dtpm_for_each_child(const struct dtpm_node *hierarchy, /** * dtpm_create_hierarchy - Create the dtpm hierarchy - * @hierarchy: An array of struct dtpm_node describing the hierarchy + * @dtpm_match_table: Pointer to the array of device ID structures * * The function is called by the platform specific code with the * description of the different node in the hierarchy. It creates the diff --git a/drivers/powercap/dtpm_cpu.c b/drivers/powercap/dtpm_cpu.c index 9193c3b8edeb..bc90126f1b5f 100644 --- a/drivers/powercap/dtpm_cpu.c +++ b/drivers/powercap/dtpm_cpu.c @@ -42,6 +42,7 @@ static u64 set_pd_power_limit(struct dtpm *dtpm, u64 power_limit) { struct dtpm_cpu *dtpm_cpu = to_dtpm_cpu(dtpm); struct em_perf_domain *pd = em_cpu_get(dtpm_cpu->cpu); + struct em_perf_state *table; struct cpumask cpus; unsigned long freq; u64 power; @@ -50,20 +51,22 @@ static u64 set_pd_power_limit(struct dtpm *dtpm, u64 power_limit) cpumask_and(&cpus, cpu_online_mask, to_cpumask(pd->cpus)); nr_cpus = cpumask_weight(&cpus); + rcu_read_lock(); + table = em_perf_state_from_pd(pd); for (i = 0; i < pd->nr_perf_states; i++) { - power = pd->table[i].power * nr_cpus; + power = table[i].power * nr_cpus; if (power > power_limit) break; } - freq = pd->table[i - 1].frequency; + freq = table[i - 1].frequency; + power_limit = table[i - 1].power * nr_cpus; + rcu_read_unlock(); freq_qos_update_request(&dtpm_cpu->qos_req, freq); - power_limit = pd->table[i - 1].power * nr_cpus; - return power_limit; } @@ -87,9 +90,11 @@ static u64 scale_pd_power_uw(struct cpumask *pd_mask, u64 power) static u64 get_pd_power_uw(struct dtpm *dtpm) { struct dtpm_cpu *dtpm_cpu = to_dtpm_cpu(dtpm); + struct em_perf_state *table; struct em_perf_domain *pd; struct cpumask *pd_mask; unsigned long freq; + u64 power = 0; int i; pd = em_cpu_get(dtpm_cpu->cpu); @@ -98,33 +103,43 @@ static u64 get_pd_power_uw(struct dtpm *dtpm) freq = cpufreq_quick_get(dtpm_cpu->cpu); + rcu_read_lock(); + table = em_perf_state_from_pd(pd); for (i = 0; i < pd->nr_perf_states; i++) { - if (pd->table[i].frequency < freq) + if (table[i].frequency < freq) continue; - return scale_pd_power_uw(pd_mask, pd->table[i].power); + power = scale_pd_power_uw(pd_mask, table[i].power); + break; } + rcu_read_unlock(); - return 0; + return power; } static int update_pd_power_uw(struct dtpm *dtpm) { struct dtpm_cpu *dtpm_cpu = to_dtpm_cpu(dtpm); struct em_perf_domain *em = em_cpu_get(dtpm_cpu->cpu); + struct em_perf_state *table; struct cpumask cpus; int nr_cpus; cpumask_and(&cpus, cpu_online_mask, to_cpumask(em->cpus)); nr_cpus = cpumask_weight(&cpus); - dtpm->power_min = em->table[0].power; + rcu_read_lock(); + table = em_perf_state_from_pd(em); + + dtpm->power_min = table[0].power; dtpm->power_min *= nr_cpus; - dtpm->power_max = em->table[em->nr_perf_states - 1].power; + dtpm->power_max = table[em->nr_perf_states - 1].power; dtpm->power_max *= nr_cpus; + rcu_read_unlock(); + return 0; } @@ -143,7 +158,7 @@ static void pd_release(struct dtpm *dtpm) cpufreq_cpu_put(policy); } - + kfree(dtpm_cpu); } @@ -180,6 +195,7 @@ static int __dtpm_cpu_setup(int cpu, struct dtpm *parent) { struct dtpm_cpu *dtpm_cpu; struct cpufreq_policy *policy; + struct em_perf_state *table; struct em_perf_domain *pd; char name[CPUFREQ_NAME_LEN]; int ret = -ENOMEM; @@ -216,10 +232,13 @@ static int __dtpm_cpu_setup(int cpu, struct dtpm *parent) if (ret) goto out_kfree_dtpm_cpu; + rcu_read_lock(); + table = em_perf_state_from_pd(pd); ret = freq_qos_add_request(&policy->constraints, &dtpm_cpu->qos_req, FREQ_QOS_MAX, - pd->table[pd->nr_perf_states - 1].frequency); - if (ret) + table[pd->nr_perf_states - 1].frequency); + rcu_read_unlock(); + if (ret < 0) goto out_dtpm_unregister; cpufreq_cpu_put(policy); diff --git a/drivers/powercap/dtpm_devfreq.c b/drivers/powercap/dtpm_devfreq.c index 612c3b59dd5b..f40bce8176df 100644 --- a/drivers/powercap/dtpm_devfreq.c +++ b/drivers/powercap/dtpm_devfreq.c @@ -37,11 +37,16 @@ static int update_pd_power_uw(struct dtpm *dtpm) struct devfreq *devfreq = dtpm_devfreq->devfreq; struct device *dev = devfreq->dev.parent; struct em_perf_domain *pd = em_pd_get(dev); + struct em_perf_state *table; - dtpm->power_min = pd->table[0].power; + rcu_read_lock(); + table = em_perf_state_from_pd(pd); - dtpm->power_max = pd->table[pd->nr_perf_states - 1].power; + dtpm->power_min = table[0].power; + dtpm->power_max = table[pd->nr_perf_states - 1].power; + + rcu_read_unlock(); return 0; } @@ -51,20 +56,23 @@ static u64 set_pd_power_limit(struct dtpm *dtpm, u64 power_limit) struct devfreq *devfreq = dtpm_devfreq->devfreq; struct device *dev = devfreq->dev.parent; struct em_perf_domain *pd = em_pd_get(dev); + struct em_perf_state *table; unsigned long freq; int i; + rcu_read_lock(); + table = em_perf_state_from_pd(pd); for (i = 0; i < pd->nr_perf_states; i++) { - if (pd->table[i].power > power_limit) + if (table[i].power > power_limit) break; } - freq = pd->table[i - 1].frequency; + freq = table[i - 1].frequency; + power_limit = table[i - 1].power; + rcu_read_unlock(); dev_pm_qos_update_request(&dtpm_devfreq->qos_req, freq); - power_limit = pd->table[i - 1].power; - return power_limit; } @@ -89,8 +97,9 @@ static u64 get_pd_power_uw(struct dtpm *dtpm) struct device *dev = devfreq->dev.parent; struct em_perf_domain *pd = em_pd_get(dev); struct devfreq_dev_status status; + struct em_perf_state *table; unsigned long freq; - u64 power; + u64 power = 0; int i; mutex_lock(&devfreq->lock); @@ -100,19 +109,22 @@ static u64 get_pd_power_uw(struct dtpm *dtpm) freq = DIV_ROUND_UP(status.current_frequency, HZ_PER_KHZ); _normalize_load(&status); + rcu_read_lock(); + table = em_perf_state_from_pd(pd); for (i = 0; i < pd->nr_perf_states; i++) { - if (pd->table[i].frequency < freq) + if (table[i].frequency < freq) continue; - power = pd->table[i].power; + power = table[i].power; power *= status.busy_time; power >>= 10; - return power; + break; } + rcu_read_unlock(); - return 0; + return power; } static void pd_release(struct dtpm *dtpm) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 00c861899a47..a28d54fd5222 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -5,6 +5,7 @@ */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -759,6 +760,11 @@ static int rapl_config(struct rapl_package *rp) default: return -EINVAL; } + + /* defaults_msr can be NULL on unsupported platforms */ + if (!rp->priv->defaults || !rp->priv->rpi) + return -ENODEV; + return 0; } @@ -1256,6 +1262,8 @@ static const struct x86_cpu_id rapl_ids[] __initconst = { X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE_L, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &rapl_defaults_spr_server), X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, &rapl_defaults_spr_server), + X86_MATCH_INTEL_FAM6_MODEL(LUNARLAKE_M, &rapl_defaults_core), + X86_MATCH_INTEL_FAM6_MODEL(ARROWLAKE, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(LAKEFIELD, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &rapl_defaults_byt), @@ -1499,7 +1507,7 @@ static int rapl_detect_domains(struct rapl_package *rp) } /* called from CPU hotplug notifier, hotplug lock held */ -void rapl_remove_package(struct rapl_package *rp) +void rapl_remove_package_cpuslocked(struct rapl_package *rp) { struct rapl_domain *rd, *rd_package = NULL; @@ -1528,10 +1536,18 @@ void rapl_remove_package(struct rapl_package *rp) list_del(&rp->plist); kfree(rp); } +EXPORT_SYMBOL_GPL(rapl_remove_package_cpuslocked); + +void rapl_remove_package(struct rapl_package *rp) +{ + guard(cpus_read_lock)(); + rapl_remove_package_cpuslocked(rp); +} EXPORT_SYMBOL_GPL(rapl_remove_package); /* caller to ensure CPU hotplug lock is held */ -struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu) +struct rapl_package *rapl_find_package_domain_cpuslocked(int id, struct rapl_if_priv *priv, + bool id_is_cpu) { struct rapl_package *rp; int uid; @@ -1549,10 +1565,17 @@ struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, return NULL; } +EXPORT_SYMBOL_GPL(rapl_find_package_domain_cpuslocked); + +struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu) +{ + guard(cpus_read_lock)(); + return rapl_find_package_domain_cpuslocked(id, priv, id_is_cpu); +} EXPORT_SYMBOL_GPL(rapl_find_package_domain); /* called from CPU hotplug notifier, hotplug lock held */ -struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu) +struct rapl_package *rapl_add_package_cpuslocked(int id, struct rapl_if_priv *priv, bool id_is_cpu) { struct rapl_package *rp; int ret; @@ -1598,6 +1621,13 @@ err_free_package: kfree(rp); return ERR_PTR(ret); } +EXPORT_SYMBOL_GPL(rapl_add_package_cpuslocked); + +struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu) +{ + guard(cpus_read_lock)(); + return rapl_add_package_cpuslocked(id, priv, id_is_cpu); +} EXPORT_SYMBOL_GPL(rapl_add_package); static void power_limit_state_save(void) diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index 250bd41a588c..b4b6930cacb0 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -73,9 +73,9 @@ static int rapl_cpu_online(unsigned int cpu) { struct rapl_package *rp; - rp = rapl_find_package_domain(cpu, rapl_msr_priv, true); + rp = rapl_find_package_domain_cpuslocked(cpu, rapl_msr_priv, true); if (!rp) { - rp = rapl_add_package(cpu, rapl_msr_priv, true); + rp = rapl_add_package_cpuslocked(cpu, rapl_msr_priv, true); if (IS_ERR(rp)) return PTR_ERR(rp); } @@ -88,14 +88,14 @@ static int rapl_cpu_down_prep(unsigned int cpu) struct rapl_package *rp; int lead_cpu; - rp = rapl_find_package_domain(cpu, rapl_msr_priv, true); + rp = rapl_find_package_domain_cpuslocked(cpu, rapl_msr_priv, true); if (!rp) return 0; cpumask_clear_cpu(cpu, &rp->cpumask); lead_cpu = cpumask_first(&rp->cpumask); if (lead_cpu >= nr_cpu_ids) - rapl_remove_package(rp); + rapl_remove_package_cpuslocked(rp); else if (rp->lead_cpu == cpu) rp->lead_cpu = lead_cpu; return 0; diff --git a/drivers/powercap/intel_rapl_tpmi.c b/drivers/powercap/intel_rapl_tpmi.c index 891c90fefd8b..f6b7f085977c 100644 --- a/drivers/powercap/intel_rapl_tpmi.c +++ b/drivers/powercap/intel_rapl_tpmi.c @@ -40,6 +40,7 @@ enum tpmi_rapl_register { TPMI_RAPL_REG_ENERGY_STATUS, TPMI_RAPL_REG_PERF_STATUS, TPMI_RAPL_REG_POWER_INFO, + TPMI_RAPL_REG_DOMAIN_INFO, TPMI_RAPL_REG_INTERRUPT, TPMI_RAPL_REG_MAX = 15, }; @@ -130,6 +131,12 @@ static void trp_release(struct tpmi_rapl_package *trp) mutex_unlock(&tpmi_rapl_lock); } +/* + * Bit 0 of TPMI_RAPL_REG_DOMAIN_INFO indicates if the current package is a domain + * root or not. Only domain root packages can enumerate System (Psys) Domain. + */ +#define TPMI_RAPL_DOMAIN_ROOT BIT(0) + static int parse_one_domain(struct tpmi_rapl_package *trp, u32 offset) { u8 tpmi_domain_version; @@ -139,6 +146,7 @@ static int parse_one_domain(struct tpmi_rapl_package *trp, u32 offset) enum rapl_domain_reg_id reg_id; int tpmi_domain_size, tpmi_domain_flags; u64 tpmi_domain_header = readq(trp->base + offset); + u64 tpmi_domain_info; /* Domain Parent bits are ignored for now */ tpmi_domain_version = tpmi_domain_header & 0xff; @@ -169,6 +177,13 @@ static int parse_one_domain(struct tpmi_rapl_package *trp, u32 offset) domain_type = RAPL_DOMAIN_PACKAGE; break; case TPMI_RAPL_DOMAIN_SYSTEM: + if (!(tpmi_domain_flags & BIT(TPMI_RAPL_REG_DOMAIN_INFO))) { + pr_warn(FW_BUG "System domain must support Domain Info register\n"); + return -ENODEV; + } + tpmi_domain_info = readq(trp->base + offset + TPMI_RAPL_REG_DOMAIN_INFO); + if (!(tpmi_domain_info & TPMI_RAPL_DOMAIN_ROOT)) + return 0; domain_type = RAPL_DOMAIN_PLATFORM; break; case TPMI_RAPL_DOMAIN_MEMORY: diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c index f2728ee787d7..d70f793ce4b3 100644 --- a/drivers/pwm/core.c +++ b/drivers/pwm/core.c @@ -24,310 +24,11 @@ #define CREATE_TRACE_POINTS #include -static DEFINE_MUTEX(pwm_lookup_lock); -static LIST_HEAD(pwm_lookup_list); - /* protects access to pwm_chips */ static DEFINE_MUTEX(pwm_lock); static DEFINE_IDR(pwm_chips); -static struct pwm_chip *pwmchip_find_by_name(const char *name) -{ - struct pwm_chip *chip; - unsigned long id, tmp; - - if (!name) - return NULL; - - mutex_lock(&pwm_lock); - - idr_for_each_entry_ul(&pwm_chips, chip, tmp, id) { - const char *chip_name = dev_name(chip->dev); - - if (chip_name && strcmp(chip_name, name) == 0) { - mutex_unlock(&pwm_lock); - return chip; - } - } - - mutex_unlock(&pwm_lock); - - return NULL; -} - -static int pwm_device_request(struct pwm_device *pwm, const char *label) -{ - int err; - struct pwm_chip *chip = pwm->chip; - const struct pwm_ops *ops = chip->ops; - - if (test_bit(PWMF_REQUESTED, &pwm->flags)) - return -EBUSY; - - if (!try_module_get(chip->owner)) - return -ENODEV; - - if (ops->request) { - err = ops->request(chip, pwm); - if (err) { - module_put(chip->owner); - return err; - } - } - - if (ops->get_state) { - /* - * Zero-initialize state because most drivers are unaware of - * .usage_power. The other members of state are supposed to be - * set by lowlevel drivers. We still initialize the whole - * structure for simplicity even though this might paper over - * faulty implementations of .get_state(). - */ - struct pwm_state state = { 0, }; - - err = ops->get_state(chip, pwm, &state); - trace_pwm_get(pwm, &state, err); - - if (!err) - pwm->state = state; - - if (IS_ENABLED(CONFIG_PWM_DEBUG)) - pwm->last = pwm->state; - } - - set_bit(PWMF_REQUESTED, &pwm->flags); - pwm->label = label; - - return 0; -} - -struct pwm_device * -of_pwm_xlate_with_flags(struct pwm_chip *chip, const struct of_phandle_args *args) -{ - struct pwm_device *pwm; - - if (chip->of_pwm_n_cells < 2) - return ERR_PTR(-EINVAL); - - /* flags in the third cell are optional */ - if (args->args_count < 2) - return ERR_PTR(-EINVAL); - - if (args->args[0] >= chip->npwm) - return ERR_PTR(-EINVAL); - - pwm = pwm_request_from_chip(chip, args->args[0], NULL); - if (IS_ERR(pwm)) - return pwm; - - pwm->args.period = args->args[1]; - pwm->args.polarity = PWM_POLARITY_NORMAL; - - if (chip->of_pwm_n_cells >= 3) { - if (args->args_count > 2 && args->args[2] & PWM_POLARITY_INVERTED) - pwm->args.polarity = PWM_POLARITY_INVERSED; - } - - return pwm; -} -EXPORT_SYMBOL_GPL(of_pwm_xlate_with_flags); - -struct pwm_device * -of_pwm_single_xlate(struct pwm_chip *chip, const struct of_phandle_args *args) -{ - struct pwm_device *pwm; - - if (chip->of_pwm_n_cells < 1) - return ERR_PTR(-EINVAL); - - /* validate that one cell is specified, optionally with flags */ - if (args->args_count != 1 && args->args_count != 2) - return ERR_PTR(-EINVAL); - - pwm = pwm_request_from_chip(chip, 0, NULL); - if (IS_ERR(pwm)) - return pwm; - - pwm->args.period = args->args[0]; - pwm->args.polarity = PWM_POLARITY_NORMAL; - - if (args->args_count == 2 && args->args[1] & PWM_POLARITY_INVERTED) - pwm->args.polarity = PWM_POLARITY_INVERSED; - - return pwm; -} -EXPORT_SYMBOL_GPL(of_pwm_single_xlate); - -static void of_pwmchip_add(struct pwm_chip *chip) -{ - if (!chip->dev || !chip->dev->of_node) - return; - - if (!chip->of_xlate) { - u32 pwm_cells; - - if (of_property_read_u32(chip->dev->of_node, "#pwm-cells", - &pwm_cells)) - pwm_cells = 2; - - chip->of_xlate = of_pwm_xlate_with_flags; - chip->of_pwm_n_cells = pwm_cells; - } - - of_node_get(chip->dev->of_node); -} - -static void of_pwmchip_remove(struct pwm_chip *chip) -{ - if (chip->dev) - of_node_put(chip->dev->of_node); -} - -static bool pwm_ops_check(const struct pwm_chip *chip) -{ - const struct pwm_ops *ops = chip->ops; - - if (!ops->apply) - return false; - - if (IS_ENABLED(CONFIG_PWM_DEBUG) && !ops->get_state) - dev_warn(chip->dev, - "Please implement the .get_state() callback\n"); - - return true; -} - -/** - * __pwmchip_add() - register a new PWM chip - * @chip: the PWM chip to add - * @owner: reference to the module providing the chip. - * - * Register a new PWM chip. @owner is supposed to be THIS_MODULE, use the - * pwmchip_add wrapper to do this right. - * - * Returns: 0 on success or a negative error code on failure. - */ -int __pwmchip_add(struct pwm_chip *chip, struct module *owner) -{ - unsigned int i; - int ret; - - if (!chip || !chip->dev || !chip->ops || !chip->npwm) - return -EINVAL; - - if (!pwm_ops_check(chip)) - return -EINVAL; - - chip->owner = owner; - - chip->pwms = kcalloc(chip->npwm, sizeof(*chip->pwms), GFP_KERNEL); - if (!chip->pwms) - return -ENOMEM; - - mutex_lock(&pwm_lock); - - ret = idr_alloc(&pwm_chips, chip, 0, 0, GFP_KERNEL); - if (ret < 0) { - mutex_unlock(&pwm_lock); - kfree(chip->pwms); - return ret; - } - - chip->id = ret; - - for (i = 0; i < chip->npwm; i++) { - struct pwm_device *pwm = &chip->pwms[i]; - - pwm->chip = chip; - pwm->hwpwm = i; - } - - mutex_unlock(&pwm_lock); - - if (IS_ENABLED(CONFIG_OF)) - of_pwmchip_add(chip); - - pwmchip_sysfs_export(chip); - - return 0; -} -EXPORT_SYMBOL_GPL(__pwmchip_add); - -/** - * pwmchip_remove() - remove a PWM chip - * @chip: the PWM chip to remove - * - * Removes a PWM chip. - */ -void pwmchip_remove(struct pwm_chip *chip) -{ - pwmchip_sysfs_unexport(chip); - - if (IS_ENABLED(CONFIG_OF)) - of_pwmchip_remove(chip); - - mutex_lock(&pwm_lock); - - idr_remove(&pwm_chips, chip->id); - - mutex_unlock(&pwm_lock); - - kfree(chip->pwms); -} -EXPORT_SYMBOL_GPL(pwmchip_remove); - -static void devm_pwmchip_remove(void *data) -{ - struct pwm_chip *chip = data; - - pwmchip_remove(chip); -} - -int __devm_pwmchip_add(struct device *dev, struct pwm_chip *chip, struct module *owner) -{ - int ret; - - ret = __pwmchip_add(chip, owner); - if (ret) - return ret; - - return devm_add_action_or_reset(dev, devm_pwmchip_remove, chip); -} -EXPORT_SYMBOL_GPL(__devm_pwmchip_add); - -/** - * pwm_request_from_chip() - request a PWM device relative to a PWM chip - * @chip: PWM chip - * @index: per-chip index of the PWM to request - * @label: a literal description string of this PWM - * - * Returns: A pointer to the PWM device at the given index of the given PWM - * chip. A negative error code is returned if the index is not valid for the - * specified PWM chip or if the PWM device cannot be requested. - */ -struct pwm_device *pwm_request_from_chip(struct pwm_chip *chip, - unsigned int index, - const char *label) -{ - struct pwm_device *pwm; - int err; - - if (!chip || index >= chip->npwm) - return ERR_PTR(-EINVAL); - - mutex_lock(&pwm_lock); - pwm = &chip->pwms[index]; - - err = pwm_device_request(pwm, label); - if (err < 0) - pwm = ERR_PTR(err); - - mutex_unlock(&pwm_lock); - return pwm; -} -EXPORT_SYMBOL_GPL(pwm_request_from_chip); - static void pwm_apply_debug(struct pwm_device *pwm, const struct pwm_state *state) { @@ -370,18 +71,18 @@ static void pwm_apply_debug(struct pwm_device *pwm, if (s2.polarity != state->polarity && state->duty_cycle < state->period) - dev_warn(chip->dev, ".apply ignored .polarity\n"); + dev_warn(pwmchip_parent(chip), ".apply ignored .polarity\n"); if (state->enabled && last->polarity == state->polarity && last->period > s2.period && last->period <= state->period) - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), ".apply didn't pick the best available period (requested: %llu, applied: %llu, possible: %llu)\n", state->period, s2.period, last->period); if (state->enabled && state->period < s2.period) - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), ".apply is supposed to round down period (requested: %llu, applied: %llu)\n", state->period, s2.period); @@ -390,20 +91,20 @@ static void pwm_apply_debug(struct pwm_device *pwm, last->period == s2.period && last->duty_cycle > s2.duty_cycle && last->duty_cycle <= state->duty_cycle) - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), ".apply didn't pick the best available duty cycle (requested: %llu/%llu, applied: %llu/%llu, possible: %llu/%llu)\n", state->duty_cycle, state->period, s2.duty_cycle, s2.period, last->duty_cycle, last->period); if (state->enabled && state->duty_cycle < s2.duty_cycle) - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), ".apply is supposed to round down duty_cycle (requested: %llu/%llu, applied: %llu/%llu)\n", state->duty_cycle, state->period, s2.duty_cycle, s2.period); if (!state->enabled && s2.enabled && s2.duty_cycle > 0) - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), "requested disabled, but yielded enabled with duty > 0\n"); /* reapply the state that the driver reported being configured. */ @@ -411,7 +112,7 @@ static void pwm_apply_debug(struct pwm_device *pwm, trace_pwm_apply(pwm, &s1, err); if (err) { *last = s1; - dev_err(chip->dev, "failed to reapply current setting\n"); + dev_err(pwmchip_parent(chip), "failed to reapply current setting\n"); return; } @@ -426,7 +127,7 @@ static void pwm_apply_debug(struct pwm_device *pwm, s1.polarity != last->polarity || (s1.enabled && s1.period != last->period) || (s1.enabled && s1.duty_cycle != last->duty_cycle)) { - dev_err(chip->dev, + dev_err(pwmchip_parent(chip), ".apply is not idempotent (ena=%d pol=%d %llu/%llu) -> (ena=%d pol=%d %llu/%llu)\n", s1.enabled, s1.polarity, s1.duty_cycle, s1.period, last->enabled, last->polarity, last->duty_cycle, @@ -523,33 +224,6 @@ int pwm_apply_atomic(struct pwm_device *pwm, const struct pwm_state *state) } EXPORT_SYMBOL_GPL(pwm_apply_atomic); -/** - * pwm_capture() - capture and report a PWM signal - * @pwm: PWM device - * @result: structure to fill with capture result - * @timeout: time to wait, in milliseconds, before giving up on capture - * - * Returns: 0 on success or a negative error code on failure. - */ -int pwm_capture(struct pwm_device *pwm, struct pwm_capture *result, - unsigned long timeout) -{ - int err; - - if (!pwm || !pwm->chip->ops) - return -EINVAL; - - if (!pwm->chip->ops->capture) - return -ENOSYS; - - mutex_lock(&pwm_lock); - err = pwm->chip->ops->capture(pwm->chip, pwm, result, timeout); - mutex_unlock(&pwm_lock); - - return err; -} -EXPORT_SYMBOL_GPL(pwm_capture); - /** * pwm_adjust_config() - adjust the current PWM config to the PWM arguments * @pwm: PWM device @@ -606,24 +280,367 @@ int pwm_adjust_config(struct pwm_device *pwm) } EXPORT_SYMBOL_GPL(pwm_adjust_config); -static struct pwm_chip *fwnode_to_pwmchip(struct fwnode_handle *fwnode) +/** + * pwm_capture() - capture and report a PWM signal + * @pwm: PWM device + * @result: structure to fill with capture result + * @timeout: time to wait, in milliseconds, before giving up on capture + * + * Returns: 0 on success or a negative error code on failure. + */ +int pwm_capture(struct pwm_device *pwm, struct pwm_capture *result, + unsigned long timeout) +{ + int err; + + if (!pwm || !pwm->chip->ops) + return -EINVAL; + + if (!pwm->chip->ops->capture) + return -ENOSYS; + + mutex_lock(&pwm_lock); + err = pwm->chip->ops->capture(pwm->chip, pwm, result, timeout); + mutex_unlock(&pwm_lock); + + return err; +} +EXPORT_SYMBOL_GPL(pwm_capture); + +static struct pwm_chip *pwmchip_find_by_name(const char *name) { struct pwm_chip *chip; unsigned long id, tmp; + if (!name) + return NULL; + mutex_lock(&pwm_lock); - idr_for_each_entry_ul(&pwm_chips, chip, tmp, id) - if (chip->dev && device_match_fwnode(chip->dev, fwnode)) { + idr_for_each_entry_ul(&pwm_chips, chip, tmp, id) { + const char *chip_name = dev_name(pwmchip_parent(chip)); + + if (chip_name && strcmp(chip_name, name) == 0) { mutex_unlock(&pwm_lock); return chip; } + } mutex_unlock(&pwm_lock); - return ERR_PTR(-EPROBE_DEFER); + return NULL; } +static int pwm_device_request(struct pwm_device *pwm, const char *label) +{ + int err; + struct pwm_chip *chip = pwm->chip; + const struct pwm_ops *ops = chip->ops; + + if (test_bit(PWMF_REQUESTED, &pwm->flags)) + return -EBUSY; + + if (!try_module_get(chip->owner)) + return -ENODEV; + + if (ops->request) { + err = ops->request(chip, pwm); + if (err) { + module_put(chip->owner); + return err; + } + } + + if (ops->get_state) { + /* + * Zero-initialize state because most drivers are unaware of + * .usage_power. The other members of state are supposed to be + * set by lowlevel drivers. We still initialize the whole + * structure for simplicity even though this might paper over + * faulty implementations of .get_state(). + */ + struct pwm_state state = { 0, }; + + err = ops->get_state(chip, pwm, &state); + trace_pwm_get(pwm, &state, err); + + if (!err) + pwm->state = state; + + if (IS_ENABLED(CONFIG_PWM_DEBUG)) + pwm->last = pwm->state; + } + + set_bit(PWMF_REQUESTED, &pwm->flags); + pwm->label = label; + + return 0; +} + +/** + * pwm_request_from_chip() - request a PWM device relative to a PWM chip + * @chip: PWM chip + * @index: per-chip index of the PWM to request + * @label: a literal description string of this PWM + * + * Returns: A pointer to the PWM device at the given index of the given PWM + * chip. A negative error code is returned if the index is not valid for the + * specified PWM chip or if the PWM device cannot be requested. + */ +struct pwm_device *pwm_request_from_chip(struct pwm_chip *chip, + unsigned int index, + const char *label) +{ + struct pwm_device *pwm; + int err; + + if (!chip || index >= chip->npwm) + return ERR_PTR(-EINVAL); + + mutex_lock(&pwm_lock); + pwm = &chip->pwms[index]; + + err = pwm_device_request(pwm, label); + if (err < 0) + pwm = ERR_PTR(err); + + mutex_unlock(&pwm_lock); + return pwm; +} +EXPORT_SYMBOL_GPL(pwm_request_from_chip); + + +struct pwm_device * +of_pwm_xlate_with_flags(struct pwm_chip *chip, const struct of_phandle_args *args) +{ + struct pwm_device *pwm; + + /* period in the second cell and flags in the third cell are optional */ + if (args->args_count < 1) + return ERR_PTR(-EINVAL); + + pwm = pwm_request_from_chip(chip, args->args[0], NULL); + if (IS_ERR(pwm)) + return pwm; + + if (args->args_count > 1) + pwm->args.period = args->args[1]; + + pwm->args.polarity = PWM_POLARITY_NORMAL; + if (args->args_count > 2 && args->args[2] & PWM_POLARITY_INVERTED) + pwm->args.polarity = PWM_POLARITY_INVERSED; + + return pwm; +} +EXPORT_SYMBOL_GPL(of_pwm_xlate_with_flags); + +struct pwm_device * +of_pwm_single_xlate(struct pwm_chip *chip, const struct of_phandle_args *args) +{ + struct pwm_device *pwm; + + pwm = pwm_request_from_chip(chip, 0, NULL); + if (IS_ERR(pwm)) + return pwm; + + if (args->args_count > 1) + pwm->args.period = args->args[0]; + + pwm->args.polarity = PWM_POLARITY_NORMAL; + if (args->args_count > 1 && args->args[1] & PWM_POLARITY_INVERTED) + pwm->args.polarity = PWM_POLARITY_INVERSED; + + return pwm; +} +EXPORT_SYMBOL_GPL(of_pwm_single_xlate); + +#define PWMCHIP_ALIGN ARCH_DMA_MINALIGN + +static void *pwmchip_priv(struct pwm_chip *chip) +{ + return (void *)chip + ALIGN(sizeof(*chip), PWMCHIP_ALIGN); +} + +/* This is the counterpart to pwmchip_alloc() */ +void pwmchip_put(struct pwm_chip *chip) +{ + kfree(chip); +} +EXPORT_SYMBOL_GPL(pwmchip_put); + +struct pwm_chip *pwmchip_alloc(struct device *parent, unsigned int npwm, size_t sizeof_priv) +{ + struct pwm_chip *chip; + size_t alloc_size; + + alloc_size = size_add(ALIGN(sizeof(*chip), PWMCHIP_ALIGN), sizeof_priv); + + chip = kzalloc(alloc_size, GFP_KERNEL); + if (!chip) + return ERR_PTR(-ENOMEM); + + chip->dev = parent; + chip->npwm = npwm; + + pwmchip_set_drvdata(chip, pwmchip_priv(chip)); + + return chip; +} +EXPORT_SYMBOL_GPL(pwmchip_alloc); + +static void devm_pwmchip_put(void *data) +{ + struct pwm_chip *chip = data; + + pwmchip_put(chip); +} + +struct pwm_chip *devm_pwmchip_alloc(struct device *parent, unsigned int npwm, size_t sizeof_priv) +{ + struct pwm_chip *chip; + int ret; + + chip = pwmchip_alloc(parent, npwm, sizeof_priv); + if (IS_ERR(chip)) + return chip; + + ret = devm_add_action_or_reset(parent, devm_pwmchip_put, chip); + if (ret) + return ERR_PTR(ret); + + return chip; +} +EXPORT_SYMBOL_GPL(devm_pwmchip_alloc); + +static void of_pwmchip_add(struct pwm_chip *chip) +{ + if (!pwmchip_parent(chip) || !pwmchip_parent(chip)->of_node) + return; + + if (!chip->of_xlate) + chip->of_xlate = of_pwm_xlate_with_flags; + + of_node_get(pwmchip_parent(chip)->of_node); +} + +static void of_pwmchip_remove(struct pwm_chip *chip) +{ + if (pwmchip_parent(chip)) + of_node_put(pwmchip_parent(chip)->of_node); +} + +static bool pwm_ops_check(const struct pwm_chip *chip) +{ + const struct pwm_ops *ops = chip->ops; + + if (!ops->apply) + return false; + + if (IS_ENABLED(CONFIG_PWM_DEBUG) && !ops->get_state) + dev_warn(pwmchip_parent(chip), + "Please implement the .get_state() callback\n"); + + return true; +} + +/** + * __pwmchip_add() - register a new PWM chip + * @chip: the PWM chip to add + * @owner: reference to the module providing the chip. + * + * Register a new PWM chip. @owner is supposed to be THIS_MODULE, use the + * pwmchip_add wrapper to do this right. + * + * Returns: 0 on success or a negative error code on failure. + */ +int __pwmchip_add(struct pwm_chip *chip, struct module *owner) +{ + unsigned int i; + int ret; + + if (!chip || !pwmchip_parent(chip) || !chip->ops || !chip->npwm) + return -EINVAL; + + if (!pwm_ops_check(chip)) + return -EINVAL; + + chip->owner = owner; + + chip->pwms = kcalloc(chip->npwm, sizeof(*chip->pwms), GFP_KERNEL); + if (!chip->pwms) + return -ENOMEM; + + mutex_lock(&pwm_lock); + + ret = idr_alloc(&pwm_chips, chip, 0, 0, GFP_KERNEL); + if (ret < 0) { + mutex_unlock(&pwm_lock); + kfree(chip->pwms); + return ret; + } + + chip->id = ret; + + for (i = 0; i < chip->npwm; i++) { + struct pwm_device *pwm = &chip->pwms[i]; + + pwm->chip = chip; + pwm->hwpwm = i; + } + + mutex_unlock(&pwm_lock); + + if (IS_ENABLED(CONFIG_OF)) + of_pwmchip_add(chip); + + pwmchip_sysfs_export(chip); + + return 0; +} +EXPORT_SYMBOL_GPL(__pwmchip_add); + +/** + * pwmchip_remove() - remove a PWM chip + * @chip: the PWM chip to remove + * + * Removes a PWM chip. + */ +void pwmchip_remove(struct pwm_chip *chip) +{ + pwmchip_sysfs_unexport(chip); + + if (IS_ENABLED(CONFIG_OF)) + of_pwmchip_remove(chip); + + mutex_lock(&pwm_lock); + + idr_remove(&pwm_chips, chip->id); + + mutex_unlock(&pwm_lock); + + kfree(chip->pwms); +} +EXPORT_SYMBOL_GPL(pwmchip_remove); + +static void devm_pwmchip_remove(void *data) +{ + struct pwm_chip *chip = data; + + pwmchip_remove(chip); +} + +int __devm_pwmchip_add(struct device *dev, struct pwm_chip *chip, struct module *owner) +{ + int ret; + + ret = __pwmchip_add(chip, owner); + if (ret) + return ret; + + return devm_add_action_or_reset(dev, devm_pwmchip_remove, chip); +} +EXPORT_SYMBOL_GPL(__devm_pwmchip_add); + static struct device_link *pwm_device_link_add(struct device *dev, struct pwm_device *pwm) { @@ -635,21 +652,39 @@ static struct device_link *pwm_device_link_add(struct device *dev, * impact the PM sequence ordering: the PWM supplier may get * suspended before the consumer. */ - dev_warn(pwm->chip->dev, + dev_warn(pwmchip_parent(pwm->chip), "No consumer device specified to create a link to\n"); return NULL; } - dl = device_link_add(dev, pwm->chip->dev, DL_FLAG_AUTOREMOVE_CONSUMER); + dl = device_link_add(dev, pwmchip_parent(pwm->chip), DL_FLAG_AUTOREMOVE_CONSUMER); if (!dl) { dev_err(dev, "failed to create device link to %s\n", - dev_name(pwm->chip->dev)); + dev_name(pwmchip_parent(pwm->chip))); return ERR_PTR(-EINVAL); } return dl; } +static struct pwm_chip *fwnode_to_pwmchip(struct fwnode_handle *fwnode) +{ + struct pwm_chip *chip; + unsigned long id, tmp; + + mutex_lock(&pwm_lock); + + idr_for_each_entry_ul(&pwm_chips, chip, tmp, id) + if (pwmchip_parent(chip) && device_match_fwnode(pwmchip_parent(chip), fwnode)) { + mutex_unlock(&pwm_lock); + return chip; + } + + mutex_unlock(&pwm_lock); + + return ERR_PTR(-EPROBE_DEFER); +} + /** * of_pwm_get() - request a PWM via the PWM framework * @dev: device for PWM consumer @@ -784,6 +819,9 @@ static struct pwm_device *acpi_pwm_get(const struct fwnode_handle *fwnode) return pwm; } +static DEFINE_MUTEX(pwm_lookup_lock); +static LIST_HEAD(pwm_lookup_list); + /** * pwm_add_table() - register PWM device consumers * @table: array of consumers to register @@ -1105,8 +1143,8 @@ static int pwm_seq_show(struct seq_file *s, void *v) seq_printf(s, "%s%d: %s/%s, %d PWM device%s\n", (char *)s->private, chip->id, - chip->dev->bus ? chip->dev->bus->name : "no-bus", - dev_name(chip->dev), chip->npwm, + pwmchip_parent(chip)->bus ? pwmchip_parent(chip)->bus->name : "no-bus", + dev_name(pwmchip_parent(chip)), chip->npwm, (chip->npwm != 1) ? "s" : ""); pwm_dbg_show(chip, s); diff --git a/drivers/pwm/pwm-ab8500.c b/drivers/pwm/pwm-ab8500.c index 670d33daea84..f000adab85b0 100644 --- a/drivers/pwm/pwm-ab8500.c +++ b/drivers/pwm/pwm-ab8500.c @@ -24,13 +24,12 @@ #define AB8500_PWM_CLKRATE 9600000 struct ab8500_pwm_chip { - struct pwm_chip chip; unsigned int hwid; }; static struct ab8500_pwm_chip *ab8500_pwm_from_chip(struct pwm_chip *chip) { - return container_of(chip, struct ab8500_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static int ab8500_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, @@ -92,12 +91,12 @@ static int ab8500_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, * when disabled. */ if (!state->enabled || duty_steps == 0) { - ret = abx500_mask_and_set_register_interruptible(chip->dev, + ret = abx500_mask_and_set_register_interruptible(pwmchip_parent(chip), AB8500_MISC, AB8500_PWM_OUT_CTRL7_REG, 1 << ab8500->hwid, 0); if (ret < 0) - dev_err(chip->dev, "%s: Failed to disable PWM, Error %d\n", + dev_err(pwmchip_parent(chip), "%s: Failed to disable PWM, Error %d\n", pwm->label, ret); return ret; } @@ -115,22 +114,22 @@ static int ab8500_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, reg = AB8500_PWM_OUT_CTRL1_REG + (ab8500->hwid * 2); - ret = abx500_set_register_interruptible(chip->dev, AB8500_MISC, + ret = abx500_set_register_interruptible(pwmchip_parent(chip), AB8500_MISC, reg, lower_val); if (ret < 0) return ret; - ret = abx500_set_register_interruptible(chip->dev, AB8500_MISC, + ret = abx500_set_register_interruptible(pwmchip_parent(chip), AB8500_MISC, (reg + 1), higher_val); if (ret < 0) return ret; /* enable */ - ret = abx500_mask_and_set_register_interruptible(chip->dev, + ret = abx500_mask_and_set_register_interruptible(pwmchip_parent(chip), AB8500_MISC, AB8500_PWM_OUT_CTRL7_REG, 1 << ab8500->hwid, 1 << ab8500->hwid); if (ret < 0) - dev_err(chip->dev, "%s: Failed to enable PWM, Error %d\n", + dev_err(pwmchip_parent(chip), "%s: Failed to enable PWM, Error %d\n", pwm->label, ret); return ret; @@ -144,7 +143,7 @@ static int ab8500_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, struct ab8500_pwm_chip *ab8500 = ab8500_pwm_from_chip(chip); unsigned int div, duty_steps; - ret = abx500_get_register_interruptible(chip->dev, AB8500_MISC, + ret = abx500_get_register_interruptible(pwmchip_parent(chip), AB8500_MISC, AB8500_PWM_OUT_CTRL7_REG, &ctrl7); if (ret) @@ -157,13 +156,13 @@ static int ab8500_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, return 0; } - ret = abx500_get_register_interruptible(chip->dev, AB8500_MISC, + ret = abx500_get_register_interruptible(pwmchip_parent(chip), AB8500_MISC, AB8500_PWM_OUT_CTRL1_REG + (ab8500->hwid * 2), &lower_val); if (ret) return ret; - ret = abx500_get_register_interruptible(chip->dev, AB8500_MISC, + ret = abx500_get_register_interruptible(pwmchip_parent(chip), AB8500_MISC, AB8500_PWM_OUT_CTRL2_REG + (ab8500->hwid * 2), &higher_val); if (ret) @@ -185,6 +184,7 @@ static const struct pwm_ops ab8500_pwm_ops = { static int ab8500_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct ab8500_pwm_chip *ab8500; int err; @@ -195,16 +195,16 @@ static int ab8500_pwm_probe(struct platform_device *pdev) * Nothing to be done in probe, this is required to get the * device which is required for ab8500 read and write */ - ab8500 = devm_kzalloc(&pdev->dev, sizeof(*ab8500), GFP_KERNEL); - if (ab8500 == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*ab8500)); + if (IS_ERR(chip)) + return PTR_ERR(chip); - ab8500->chip.dev = &pdev->dev; - ab8500->chip.ops = &ab8500_pwm_ops; - ab8500->chip.npwm = 1; + ab8500 = ab8500_pwm_from_chip(chip); + + chip->ops = &ab8500_pwm_ops; ab8500->hwid = pdev->id - 1; - err = devm_pwmchip_add(&pdev->dev, &ab8500->chip); + err = devm_pwmchip_add(&pdev->dev, chip); if (err < 0) return dev_err_probe(&pdev->dev, err, "Failed to add pwm chip\n"); diff --git a/drivers/pwm/pwm-apple.c b/drivers/pwm/pwm-apple.c index 4d755b628d9e..6e58aca2f13c 100644 --- a/drivers/pwm/pwm-apple.c +++ b/drivers/pwm/pwm-apple.c @@ -32,14 +32,13 @@ #define APPLE_PWM_CTRL_OUTPUT_ENABLE BIT(14) struct apple_pwm { - struct pwm_chip chip; void __iomem *base; u64 clkrate; }; static inline struct apple_pwm *to_apple_pwm(struct pwm_chip *chip) { - return container_of(chip, struct apple_pwm, chip); + return pwmchip_get_drvdata(chip); } static int apple_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, @@ -103,13 +102,16 @@ static const struct pwm_ops apple_pwm_ops = { static int apple_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct apple_pwm *fpwm; struct clk *clk; int ret; - fpwm = devm_kzalloc(&pdev->dev, sizeof(*fpwm), GFP_KERNEL); - if (!fpwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*fpwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + + fpwm = to_apple_pwm(chip); fpwm->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(fpwm->base)) @@ -129,11 +131,9 @@ static int apple_pwm_probe(struct platform_device *pdev) if (fpwm->clkrate > NSEC_PER_SEC) return dev_err_probe(&pdev->dev, -EINVAL, "pwm clock out of range"); - fpwm->chip.dev = &pdev->dev; - fpwm->chip.npwm = 1; - fpwm->chip.ops = &apple_pwm_ops; + chip->ops = &apple_pwm_ops; - ret = devm_pwmchip_add(&pdev->dev, &fpwm->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "unable to add pwm chip"); diff --git a/drivers/pwm/pwm-atmel-hlcdc.c b/drivers/pwm/pwm-atmel-hlcdc.c index 3f2c5031a3ba..2afb302be02c 100644 --- a/drivers/pwm/pwm-atmel-hlcdc.c +++ b/drivers/pwm/pwm-atmel-hlcdc.c @@ -28,7 +28,6 @@ struct atmel_hlcdc_pwm_errata { }; struct atmel_hlcdc_pwm { - struct pwm_chip chip; struct atmel_hlcdc *hlcdc; struct clk *cur_clk; const struct atmel_hlcdc_pwm_errata *errata; @@ -36,7 +35,7 @@ struct atmel_hlcdc_pwm { static inline struct atmel_hlcdc_pwm *to_atmel_hlcdc_pwm(struct pwm_chip *chip) { - return container_of(chip, struct atmel_hlcdc_pwm, chip); + return pwmchip_get_drvdata(chip); } static int atmel_hlcdc_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, @@ -182,10 +181,12 @@ static const struct atmel_hlcdc_pwm_errata atmel_hlcdc_pwm_sama5d3_errata = { static int atmel_hlcdc_pwm_suspend(struct device *dev) { - struct atmel_hlcdc_pwm *atmel = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct atmel_hlcdc_pwm *atmel = to_atmel_hlcdc_pwm(chip); + struct pwm_device *pwm = &chip->pwms[0]; /* Keep the periph clock enabled if the PWM is still running. */ - if (pwm_is_enabled(&atmel->chip.pwms[0])) + if (!pwm->state.enabled) clk_disable_unprepare(atmel->hlcdc->periph_clk); return 0; @@ -193,21 +194,19 @@ static int atmel_hlcdc_pwm_suspend(struct device *dev) static int atmel_hlcdc_pwm_resume(struct device *dev) { - struct atmel_hlcdc_pwm *atmel = dev_get_drvdata(dev); - struct pwm_state state; + struct pwm_chip *chip = dev_get_drvdata(dev); + struct atmel_hlcdc_pwm *atmel = to_atmel_hlcdc_pwm(chip); + struct pwm_device *pwm = &chip->pwms[0]; int ret; - pwm_get_state(&atmel->chip.pwms[0], &state); - /* Re-enable the periph clock it was stopped during suspend. */ - if (!state.enabled) { + if (!pwm->state.enabled) { ret = clk_prepare_enable(atmel->hlcdc->periph_clk); if (ret) return ret; } - return atmel_hlcdc_pwm_apply(&atmel->chip, &atmel->chip.pwms[0], - &state); + return atmel_hlcdc_pwm_apply(chip, pwm, &pwm->state); } static DEFINE_SIMPLE_DEV_PM_OPS(atmel_hlcdc_pwm_pm_ops, @@ -243,15 +242,17 @@ static int atmel_hlcdc_pwm_probe(struct platform_device *pdev) { const struct of_device_id *match; struct device *dev = &pdev->dev; + struct pwm_chip *chip; struct atmel_hlcdc_pwm *atmel; struct atmel_hlcdc *hlcdc; int ret; hlcdc = dev_get_drvdata(dev->parent); - atmel = devm_kzalloc(dev, sizeof(*atmel), GFP_KERNEL); - if (!atmel) - return -ENOMEM; + chip = devm_pwmchip_alloc(dev, 1, sizeof(*atmel)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + atmel = to_atmel_hlcdc_pwm(chip); ret = clk_prepare_enable(hlcdc->periph_clk); if (ret) @@ -262,26 +263,25 @@ static int atmel_hlcdc_pwm_probe(struct platform_device *pdev) atmel->errata = match->data; atmel->hlcdc = hlcdc; - atmel->chip.ops = &atmel_hlcdc_pwm_ops; - atmel->chip.dev = dev; - atmel->chip.npwm = 1; + chip->ops = &atmel_hlcdc_pwm_ops; - ret = pwmchip_add(&atmel->chip); + ret = pwmchip_add(chip); if (ret) { clk_disable_unprepare(hlcdc->periph_clk); return ret; } - platform_set_drvdata(pdev, atmel); + platform_set_drvdata(pdev, chip); return 0; } static void atmel_hlcdc_pwm_remove(struct platform_device *pdev) { - struct atmel_hlcdc_pwm *atmel = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct atmel_hlcdc_pwm *atmel = to_atmel_hlcdc_pwm(chip); - pwmchip_remove(&atmel->chip); + pwmchip_remove(chip); clk_disable_unprepare(atmel->hlcdc->periph_clk); } diff --git a/drivers/pwm/pwm-atmel-tcb.c b/drivers/pwm/pwm-atmel-tcb.c index d42c897cb85e..528e54c5999d 100644 --- a/drivers/pwm/pwm-atmel-tcb.c +++ b/drivers/pwm/pwm-atmel-tcb.c @@ -47,7 +47,6 @@ struct atmel_tcb_channel { }; struct atmel_tcb_pwm_chip { - struct pwm_chip chip; spinlock_t lock; u8 channel; u8 width; @@ -63,7 +62,7 @@ static const u8 atmel_tcb_divisors[] = { 2, 8, 32, 128, 0, }; static inline struct atmel_tcb_pwm_chip *to_tcb_chip(struct pwm_chip *chip) { - return container_of(chip, struct atmel_tcb_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static int atmel_tcb_pwm_request(struct pwm_chip *chip, @@ -327,7 +326,7 @@ static int atmel_tcb_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if ((atcbpwm && atcbpwm->duty > 0 && atcbpwm->duty != atcbpwm->period) && (atcbpwm->div != i || atcbpwm->period != period)) { - dev_err(chip->dev, + dev_err(pwmchip_parent(chip), "failed to configure period_ns: PWM group already configured with a different value\n"); return -EINVAL; } @@ -388,6 +387,7 @@ static const struct of_device_id atmel_tcb_of_match[] = { static int atmel_tcb_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; const struct of_device_id *match; struct atmel_tcb_pwm_chip *tcbpwm; const struct atmel_tcb_config *config; @@ -396,9 +396,10 @@ static int atmel_tcb_pwm_probe(struct platform_device *pdev) int err; int channel; - tcbpwm = devm_kzalloc(&pdev->dev, sizeof(*tcbpwm), GFP_KERNEL); - if (tcbpwm == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, NPWM, sizeof(*tcbpwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + tcbpwm = to_tcb_chip(chip); err = of_property_read_u32(np, "reg", &channel); if (err < 0) { @@ -436,9 +437,7 @@ static int atmel_tcb_pwm_probe(struct platform_device *pdev) } } - tcbpwm->chip.dev = &pdev->dev; - tcbpwm->chip.ops = &atmel_tcb_pwm_ops; - tcbpwm->chip.npwm = NPWM; + chip->ops = &atmel_tcb_pwm_ops; tcbpwm->channel = channel; tcbpwm->width = config->counter_width; @@ -448,11 +447,11 @@ static int atmel_tcb_pwm_probe(struct platform_device *pdev) spin_lock_init(&tcbpwm->lock); - err = pwmchip_add(&tcbpwm->chip); + err = pwmchip_add(chip); if (err < 0) goto err_disable_clk; - platform_set_drvdata(pdev, tcbpwm); + platform_set_drvdata(pdev, chip); return 0; @@ -473,9 +472,10 @@ err_slow_clk: static void atmel_tcb_pwm_remove(struct platform_device *pdev) { - struct atmel_tcb_pwm_chip *tcbpwm = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct atmel_tcb_pwm_chip *tcbpwm = to_tcb_chip(chip); - pwmchip_remove(&tcbpwm->chip); + pwmchip_remove(chip); clk_disable_unprepare(tcbpwm->slow_clk); clk_put(tcbpwm->gclk); @@ -491,7 +491,8 @@ MODULE_DEVICE_TABLE(of, atmel_tcb_pwm_dt_ids); static int atmel_tcb_pwm_suspend(struct device *dev) { - struct atmel_tcb_pwm_chip *tcbpwm = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct atmel_tcb_pwm_chip *tcbpwm = to_tcb_chip(chip); struct atmel_tcb_channel *chan = &tcbpwm->bkup; unsigned int channel = tcbpwm->channel; @@ -505,7 +506,8 @@ static int atmel_tcb_pwm_suspend(struct device *dev) static int atmel_tcb_pwm_resume(struct device *dev) { - struct atmel_tcb_pwm_chip *tcbpwm = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct atmel_tcb_pwm_chip *tcbpwm = to_tcb_chip(chip); struct atmel_tcb_channel *chan = &tcbpwm->bkup; unsigned int channel = tcbpwm->channel; diff --git a/drivers/pwm/pwm-atmel.c b/drivers/pwm/pwm-atmel.c index 47bcc8a3bf9d..b2f0abbbad63 100644 --- a/drivers/pwm/pwm-atmel.c +++ b/drivers/pwm/pwm-atmel.c @@ -77,7 +77,6 @@ struct atmel_pwm_data { }; struct atmel_pwm_chip { - struct pwm_chip chip; struct clk *clk; void __iomem *base; const struct atmel_pwm_data *data; @@ -99,7 +98,7 @@ struct atmel_pwm_chip { static inline struct atmel_pwm_chip *to_atmel_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct atmel_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static inline u32 atmel_pwm_readl(struct atmel_pwm_chip *chip, @@ -210,7 +209,7 @@ static int atmel_pwm_calculate_cprd_and_pres(struct pwm_chip *chip, shift = fls(cycles) - atmel_pwm->data->cfg.period_bits; if (shift > PWM_MAX_PRES) { - dev_err(chip->dev, "pres exceeds the maximum value\n"); + dev_err(pwmchip_parent(chip), "pres exceeds the maximum value\n"); return -EINVAL; } else if (shift > 0) { *pres = shift; @@ -294,19 +293,16 @@ static int atmel_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { struct atmel_pwm_chip *atmel_pwm = to_atmel_pwm_chip(chip); - struct pwm_state cstate; unsigned long cprd, cdty; u32 pres, val; int ret; - pwm_get_state(pwm, &cstate); - if (state->enabled) { unsigned long clkrate = clk_get_rate(atmel_pwm->clk); - if (cstate.enabled && - cstate.polarity == state->polarity && - cstate.period == state->period) { + if (pwm->state.enabled && + pwm->state.polarity == state->polarity && + pwm->state.period == state->period) { u32 cmr = atmel_pwm_ch_readl(atmel_pwm, pwm->hwpwm, PWM_CMR); cprd = atmel_pwm_ch_readl(atmel_pwm, pwm->hwpwm, @@ -321,19 +317,19 @@ static int atmel_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, ret = atmel_pwm_calculate_cprd_and_pres(chip, clkrate, state, &cprd, &pres); if (ret) { - dev_err(chip->dev, + dev_err(pwmchip_parent(chip), "failed to calculate cprd and prescaler\n"); return ret; } atmel_pwm_calculate_cdty(state, clkrate, cprd, pres, &cdty); - if (cstate.enabled) { + if (pwm->state.enabled) { atmel_pwm_disable(chip, pwm, false); } else { ret = clk_enable(atmel_pwm->clk); if (ret) { - dev_err(chip->dev, "failed to enable clock\n"); + dev_err(pwmchip_parent(chip), "failed to enable clock\n"); return ret; } } @@ -348,7 +344,7 @@ static int atmel_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, atmel_pwm_ch_writel(atmel_pwm, pwm->hwpwm, PWM_CMR, val); atmel_pwm_set_cprd_cdty(chip, pwm, cprd, cdty); atmel_pwm_writel(atmel_pwm, PWM_ENA, 1 << pwm->hwpwm); - } else if (cstate.enabled) { + } else if (pwm->state.enabled) { atmel_pwm_disable(chip, pwm, true); } @@ -462,8 +458,9 @@ static const struct of_device_id atmel_pwm_dt_ids[] = { }; MODULE_DEVICE_TABLE(of, atmel_pwm_dt_ids); -static int atmel_pwm_enable_clk_if_on(struct atmel_pwm_chip *atmel_pwm, bool on) +static int atmel_pwm_enable_clk_if_on(struct pwm_chip *chip, bool on) { + struct atmel_pwm_chip *atmel_pwm = to_atmel_pwm_chip(chip); unsigned int i, cnt = 0; unsigned long sr; int ret = 0; @@ -472,7 +469,7 @@ static int atmel_pwm_enable_clk_if_on(struct atmel_pwm_chip *atmel_pwm, bool on) if (!sr) return 0; - cnt = bitmap_weight(&sr, atmel_pwm->chip.npwm); + cnt = bitmap_weight(&sr, chip->npwm); if (!on) goto disable_clk; @@ -480,7 +477,7 @@ static int atmel_pwm_enable_clk_if_on(struct atmel_pwm_chip *atmel_pwm, bool on) for (i = 0; i < cnt; i++) { ret = clk_enable(atmel_pwm->clk); if (ret) { - dev_err(atmel_pwm->chip.dev, + dev_err(pwmchip_parent(chip), "failed to enable clock for pwm %pe\n", ERR_PTR(ret)); @@ -501,12 +498,14 @@ disable_clk: static int atmel_pwm_probe(struct platform_device *pdev) { struct atmel_pwm_chip *atmel_pwm; + struct pwm_chip *chip; int ret; - atmel_pwm = devm_kzalloc(&pdev->dev, sizeof(*atmel_pwm), GFP_KERNEL); - if (!atmel_pwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 4, sizeof(*atmel_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + atmel_pwm = to_atmel_pwm_chip(chip); atmel_pwm->data = of_device_get_match_data(&pdev->dev); atmel_pwm->update_pending = 0; @@ -521,15 +520,13 @@ static int atmel_pwm_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(atmel_pwm->clk), "failed to get prepared PWM clock\n"); - atmel_pwm->chip.dev = &pdev->dev; - atmel_pwm->chip.ops = &atmel_pwm_ops; - atmel_pwm->chip.npwm = 4; + chip->ops = &atmel_pwm_ops; - ret = atmel_pwm_enable_clk_if_on(atmel_pwm, true); + ret = atmel_pwm_enable_clk_if_on(chip, true); if (ret < 0) return ret; - ret = devm_pwmchip_add(&pdev->dev, &atmel_pwm->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) { dev_err_probe(&pdev->dev, ret, "failed to add PWM chip\n"); goto disable_clk; @@ -538,7 +535,7 @@ static int atmel_pwm_probe(struct platform_device *pdev) return 0; disable_clk: - atmel_pwm_enable_clk_if_on(atmel_pwm, false); + atmel_pwm_enable_clk_if_on(chip, false); return ret; } diff --git a/drivers/pwm/pwm-bcm-iproc.c b/drivers/pwm/pwm-bcm-iproc.c index 758254025683..f4c9f10e490e 100644 --- a/drivers/pwm/pwm-bcm-iproc.c +++ b/drivers/pwm/pwm-bcm-iproc.c @@ -34,14 +34,13 @@ #define IPROC_PWM_PRESCALE_MAX 0x3f struct iproc_pwmc { - struct pwm_chip chip; void __iomem *base; struct clk *clk; }; static inline struct iproc_pwmc *to_iproc_pwmc(struct pwm_chip *chip) { - return container_of(chip, struct iproc_pwmc, chip); + return pwmchip_get_drvdata(chip); } static void iproc_pwmc_enable(struct iproc_pwmc *ip, unsigned int channel) @@ -187,20 +186,20 @@ static const struct pwm_ops iproc_pwm_ops = { static int iproc_pwmc_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct iproc_pwmc *ip; unsigned int i; u32 value; int ret; - ip = devm_kzalloc(&pdev->dev, sizeof(*ip), GFP_KERNEL); - if (!ip) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 4, sizeof(*ip)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + ip = to_iproc_pwmc(chip); platform_set_drvdata(pdev, ip); - ip->chip.dev = &pdev->dev; - ip->chip.ops = &iproc_pwm_ops; - ip->chip.npwm = 4; + chip->ops = &iproc_pwm_ops; ip->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(ip->base)) @@ -214,14 +213,14 @@ static int iproc_pwmc_probe(struct platform_device *pdev) /* Set full drive and normal polarity for all channels */ value = readl(ip->base + IPROC_PWM_CTRL_OFFSET); - for (i = 0; i < ip->chip.npwm; i++) { + for (i = 0; i < chip->npwm; i++) { value &= ~(1 << IPROC_PWM_CTRL_TYPE_SHIFT(i)); value |= 1 << IPROC_PWM_CTRL_POLARITY_SHIFT(i); } writel(value, ip->base + IPROC_PWM_CTRL_OFFSET); - ret = devm_pwmchip_add(&pdev->dev, &ip->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "failed to add PWM chip\n"); diff --git a/drivers/pwm/pwm-bcm-kona.c b/drivers/pwm/pwm-bcm-kona.c index 45046a5c20a5..022c078aae84 100644 --- a/drivers/pwm/pwm-bcm-kona.c +++ b/drivers/pwm/pwm-bcm-kona.c @@ -56,14 +56,13 @@ #define DUTY_CYCLE_HIGH_MAX 0x00ffffff struct kona_pwmc { - struct pwm_chip chip; void __iomem *base; struct clk *clk; }; static inline struct kona_pwmc *to_kona_pwmc(struct pwm_chip *chip) { - return container_of(chip, struct kona_pwmc, chip); + return pwmchip_get_drvdata(chip); } /* @@ -164,7 +163,7 @@ static int kona_pwmc_set_polarity(struct pwm_chip *chip, struct pwm_device *pwm, ret = clk_prepare_enable(kp->clk); if (ret < 0) { - dev_err(chip->dev, "failed to enable clock: %d\n", ret); + dev_err(pwmchip_parent(chip), "failed to enable clock: %d\n", ret); return ret; } @@ -193,7 +192,7 @@ static int kona_pwmc_enable(struct pwm_chip *chip, struct pwm_device *pwm) ret = clk_prepare_enable(kp->clk); if (ret < 0) { - dev_err(chip->dev, "failed to enable clock: %d\n", ret); + dev_err(pwmchip_parent(chip), "failed to enable clock: %d\n", ret); return ret; } @@ -273,18 +272,18 @@ static const struct pwm_ops kona_pwm_ops = { static int kona_pwmc_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct kona_pwmc *kp; unsigned int chan; unsigned int value = 0; int ret = 0; - kp = devm_kzalloc(&pdev->dev, sizeof(*kp), GFP_KERNEL); - if (kp == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 6, sizeof(*kp)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + kp = to_kona_pwmc(chip); - kp->chip.dev = &pdev->dev; - kp->chip.ops = &kona_pwm_ops; - kp->chip.npwm = 6; + chip->ops = &kona_pwm_ops; kp->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(kp->base)) @@ -304,14 +303,14 @@ static int kona_pwmc_probe(struct platform_device *pdev) } /* Set push/pull for all channels */ - for (chan = 0; chan < kp->chip.npwm; chan++) + for (chan = 0; chan < chip->npwm; chan++) value |= (1 << PWM_CONTROL_TYPE_SHIFT(chan)); writel(value, kp->base + PWM_CONTROL_OFFSET); clk_disable_unprepare(kp->clk); - ret = devm_pwmchip_add(&pdev->dev, &kp->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) dev_err(&pdev->dev, "failed to add PWM chip: %d\n", ret); diff --git a/drivers/pwm/pwm-bcm2835.c b/drivers/pwm/pwm-bcm2835.c index 283cf27f25ba..aa35acbb0cbc 100644 --- a/drivers/pwm/pwm-bcm2835.c +++ b/drivers/pwm/pwm-bcm2835.c @@ -24,8 +24,6 @@ #define PERIOD_MIN 0x2 struct bcm2835_pwm { - struct pwm_chip chip; - struct device *dev; void __iomem *base; struct clk *clk; unsigned long rate; @@ -33,7 +31,7 @@ struct bcm2835_pwm { static inline struct bcm2835_pwm *to_bcm2835_pwm(struct pwm_chip *chip) { - return container_of(chip, struct bcm2835_pwm, chip); + return pwmchip_get_drvdata(chip); } static int bcm2835_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) @@ -135,14 +133,14 @@ static void devm_clk_rate_exclusive_put(void *data) static int bcm2835_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct bcm2835_pwm *pc; int ret; - pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL); - if (!pc) - return -ENOMEM; - - pc->dev = &pdev->dev; + chip = devm_pwmchip_alloc(&pdev->dev, 2, sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_bcm2835_pwm(chip); pc->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->base)) @@ -168,14 +166,12 @@ static int bcm2835_pwm_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, -EINVAL, "failed to get clock rate\n"); - pc->chip.dev = &pdev->dev; - pc->chip.ops = &bcm2835_pwm_ops; - pc->chip.atomic = true; - pc->chip.npwm = 2; + chip->ops = &bcm2835_pwm_ops; + chip->atomic = true; platform_set_drvdata(pdev, pc); - ret = devm_pwmchip_add(&pdev->dev, &pc->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "failed to add pwmchip\n"); diff --git a/drivers/pwm/pwm-berlin.c b/drivers/pwm/pwm-berlin.c index 442913232dc0..831aed228caf 100644 --- a/drivers/pwm/pwm-berlin.c +++ b/drivers/pwm/pwm-berlin.c @@ -49,7 +49,6 @@ struct berlin_pwm_channel { }; struct berlin_pwm_chip { - struct pwm_chip chip; struct clk *clk; void __iomem *base; struct berlin_pwm_channel channel[BERLIN_PWM_NUMPWMS]; @@ -57,7 +56,7 @@ struct berlin_pwm_chip { static inline struct berlin_pwm_chip *to_berlin_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct berlin_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static inline u32 berlin_pwm_readl(struct berlin_pwm_chip *bpc, @@ -198,12 +197,14 @@ MODULE_DEVICE_TABLE(of, berlin_pwm_match); static int berlin_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct berlin_pwm_chip *bpc; int ret; - bpc = devm_kzalloc(&pdev->dev, sizeof(*bpc), GFP_KERNEL); - if (!bpc) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, BERLIN_PWM_NUMPWMS, sizeof(*bpc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + bpc = to_berlin_pwm_chip(chip); bpc->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(bpc->base)) @@ -213,25 +214,24 @@ static int berlin_pwm_probe(struct platform_device *pdev) if (IS_ERR(bpc->clk)) return PTR_ERR(bpc->clk); - bpc->chip.dev = &pdev->dev; - bpc->chip.ops = &berlin_pwm_ops; - bpc->chip.npwm = BERLIN_PWM_NUMPWMS; + chip->ops = &berlin_pwm_ops; - ret = devm_pwmchip_add(&pdev->dev, &bpc->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "failed to add PWM chip\n"); - platform_set_drvdata(pdev, bpc); + platform_set_drvdata(pdev, chip); return 0; } static int berlin_pwm_suspend(struct device *dev) { - struct berlin_pwm_chip *bpc = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct berlin_pwm_chip *bpc = to_berlin_pwm_chip(chip); unsigned int i; - for (i = 0; i < bpc->chip.npwm; i++) { + for (i = 0; i < chip->npwm; i++) { struct berlin_pwm_channel *channel = &bpc->channel[i]; channel->enable = berlin_pwm_readl(bpc, i, BERLIN_PWM_ENABLE); @@ -247,7 +247,8 @@ static int berlin_pwm_suspend(struct device *dev) static int berlin_pwm_resume(struct device *dev) { - struct berlin_pwm_chip *bpc = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct berlin_pwm_chip *bpc = to_berlin_pwm_chip(chip); unsigned int i; int ret; @@ -255,7 +256,7 @@ static int berlin_pwm_resume(struct device *dev) if (ret) return ret; - for (i = 0; i < bpc->chip.npwm; i++) { + for (i = 0; i < chip->npwm; i++) { struct berlin_pwm_channel *channel = &bpc->channel[i]; berlin_pwm_writel(bpc, i, channel->ctrl, BERLIN_PWM_CONTROL); diff --git a/drivers/pwm/pwm-brcmstb.c b/drivers/pwm/pwm-brcmstb.c index 0fdeb0b2dbf3..82d27d07ba91 100644 --- a/drivers/pwm/pwm-brcmstb.c +++ b/drivers/pwm/pwm-brcmstb.c @@ -54,7 +54,6 @@ struct brcmstb_pwm { void __iomem *base; struct clk *clk; - struct pwm_chip chip; }; static inline u32 brcmstb_pwm_readl(struct brcmstb_pwm *p, @@ -77,7 +76,7 @@ static inline void brcmstb_pwm_writel(struct brcmstb_pwm *p, u32 value, static inline struct brcmstb_pwm *to_brcmstb_pwm(struct pwm_chip *chip) { - return container_of(chip, struct brcmstb_pwm, chip); + return pwmchip_get_drvdata(chip); } /* @@ -230,12 +229,14 @@ MODULE_DEVICE_TABLE(of, brcmstb_pwm_of_match); static int brcmstb_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct brcmstb_pwm *p; int ret; - p = devm_kzalloc(&pdev->dev, sizeof(*p), GFP_KERNEL); - if (!p) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 2, sizeof(*p)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + p = to_brcmstb_pwm(chip); p->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(p->clk)) @@ -244,15 +245,13 @@ static int brcmstb_pwm_probe(struct platform_device *pdev) platform_set_drvdata(pdev, p); - p->chip.dev = &pdev->dev; - p->chip.ops = &brcmstb_pwm_ops; - p->chip.npwm = 2; + chip->ops = &brcmstb_pwm_ops; p->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(p->base)) return PTR_ERR(p->base); - ret = devm_pwmchip_add(&pdev->dev, &p->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret) return dev_err_probe(&pdev->dev, ret, "failed to add PWM chip\n"); diff --git a/drivers/pwm/pwm-clk.c b/drivers/pwm/pwm-clk.c index 9dd88b386907..c19a482d7e28 100644 --- a/drivers/pwm/pwm-clk.c +++ b/drivers/pwm/pwm-clk.c @@ -28,12 +28,14 @@ #include struct pwm_clk_chip { - struct pwm_chip chip; struct clk *clk; bool clk_enabled; }; -#define to_pwm_clk_chip(_chip) container_of(_chip, struct pwm_clk_chip, chip) +static inline struct pwm_clk_chip *to_pwm_clk_chip(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} static int pwm_clk_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) @@ -81,35 +83,36 @@ static const struct pwm_ops pwm_clk_ops = { static int pwm_clk_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct pwm_clk_chip *pcchip; int ret; - pcchip = devm_kzalloc(&pdev->dev, sizeof(*pcchip), GFP_KERNEL); - if (!pcchip) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*pcchip)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pcchip = to_pwm_clk_chip(chip); pcchip->clk = devm_clk_get_prepared(&pdev->dev, NULL); if (IS_ERR(pcchip->clk)) return dev_err_probe(&pdev->dev, PTR_ERR(pcchip->clk), "Failed to get clock\n"); - pcchip->chip.dev = &pdev->dev; - pcchip->chip.ops = &pwm_clk_ops; - pcchip->chip.npwm = 1; + chip->ops = &pwm_clk_ops; - ret = pwmchip_add(&pcchip->chip); + ret = pwmchip_add(chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "Failed to add pwm chip\n"); - platform_set_drvdata(pdev, pcchip); + platform_set_drvdata(pdev, chip); return 0; } static void pwm_clk_remove(struct platform_device *pdev) { - struct pwm_clk_chip *pcchip = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct pwm_clk_chip *pcchip = to_pwm_clk_chip(chip); - pwmchip_remove(&pcchip->chip); + pwmchip_remove(chip); if (pcchip->clk_enabled) clk_disable(pcchip->clk); diff --git a/drivers/pwm/pwm-clps711x.c b/drivers/pwm/pwm-clps711x.c index 42179b3f7ec3..c950e1dbd2b8 100644 --- a/drivers/pwm/pwm-clps711x.c +++ b/drivers/pwm/pwm-clps711x.c @@ -12,7 +12,6 @@ #include struct clps711x_chip { - struct pwm_chip chip; void __iomem *pmpcon; struct clk *clk; spinlock_t lock; @@ -20,7 +19,7 @@ struct clps711x_chip { static inline struct clps711x_chip *to_clps711x_chip(struct pwm_chip *chip) { - return container_of(chip, struct clps711x_chip, chip); + return pwmchip_get_drvdata(chip); } static int clps711x_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) @@ -74,22 +73,15 @@ static const struct pwm_ops clps711x_pwm_ops = { .apply = clps711x_pwm_apply, }; -static struct pwm_device *clps711x_pwm_xlate(struct pwm_chip *chip, - const struct of_phandle_args *args) -{ - if (args->args[0] >= chip->npwm) - return ERR_PTR(-EINVAL); - - return pwm_request_from_chip(chip, args->args[0], NULL); -} - static int clps711x_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct clps711x_chip *priv; - priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 2, sizeof(*priv)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + priv = to_clps711x_chip(chip); priv->pmpcon = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(priv->pmpcon)) @@ -99,15 +91,11 @@ static int clps711x_pwm_probe(struct platform_device *pdev) if (IS_ERR(priv->clk)) return PTR_ERR(priv->clk); - priv->chip.ops = &clps711x_pwm_ops; - priv->chip.dev = &pdev->dev; - priv->chip.npwm = 2; - priv->chip.of_xlate = clps711x_pwm_xlate; - priv->chip.of_pwm_n_cells = 1; + chip->ops = &clps711x_pwm_ops; spin_lock_init(&priv->lock); - return devm_pwmchip_add(&pdev->dev, &priv->chip); + return devm_pwmchip_add(&pdev->dev, chip); } static const struct of_device_id __maybe_unused clps711x_pwm_dt_ids[] = { diff --git a/drivers/pwm/pwm-crc.c b/drivers/pwm/pwm-crc.c index e09358901ab5..98ee5cdbd0ba 100644 --- a/drivers/pwm/pwm-crc.c +++ b/drivers/pwm/pwm-crc.c @@ -26,17 +26,15 @@ /** * struct crystalcove_pwm - Crystal Cove PWM controller - * @chip: the abstract pwm_chip structure. * @regmap: the regmap from the parent device. */ struct crystalcove_pwm { - struct pwm_chip chip; struct regmap *regmap; }; static inline struct crystalcove_pwm *to_crc_pwm(struct pwm_chip *chip) { - return container_of(chip, struct crystalcove_pwm, chip); + return pwmchip_get_drvdata(chip); } static int crc_pwm_calc_clk_div(int period_ns) @@ -55,7 +53,7 @@ static int crc_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { struct crystalcove_pwm *crc_pwm = to_crc_pwm(chip); - struct device *dev = crc_pwm->chip.dev; + struct device *dev = pwmchip_parent(chip); int err; if (state->period > PWM_MAX_PERIOD_NS) { @@ -125,7 +123,7 @@ static int crc_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, struct pwm_state *state) { struct crystalcove_pwm *crc_pwm = to_crc_pwm(chip); - struct device *dev = crc_pwm->chip.dev; + struct device *dev = pwmchip_parent(chip); unsigned int clk_div, clk_div_reg, duty_cycle_reg; int error; @@ -160,22 +158,22 @@ static const struct pwm_ops crc_pwm_ops = { static int crystalcove_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct crystalcove_pwm *crc_pwm; struct device *dev = pdev->dev.parent; struct intel_soc_pmic *pmic = dev_get_drvdata(dev); - crc_pwm = devm_kzalloc(&pdev->dev, sizeof(*crc_pwm), GFP_KERNEL); - if (!crc_pwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*crc_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + crc_pwm = to_crc_pwm(chip); - crc_pwm->chip.dev = &pdev->dev; - crc_pwm->chip.ops = &crc_pwm_ops; - crc_pwm->chip.npwm = 1; + chip->ops = &crc_pwm_ops; /* get the PMIC regmap */ crc_pwm->regmap = pmic->regmap; - return devm_pwmchip_add(&pdev->dev, &crc_pwm->chip); + return devm_pwmchip_add(&pdev->dev, chip); } static struct platform_driver crystalcove_pwm_driver = { diff --git a/drivers/pwm/pwm-cros-ec.c b/drivers/pwm/pwm-cros-ec.c index 5fe303b8656d..606ccfdaf4cc 100644 --- a/drivers/pwm/pwm-cros-ec.c +++ b/drivers/pwm/pwm-cros-ec.c @@ -19,13 +19,11 @@ * struct cros_ec_pwm_device - Driver data for EC PWM * * @ec: Pointer to EC device - * @chip: PWM controller chip * @use_pwm_type: Use PWM types instead of generic channels * @channel: array with per-channel data */ struct cros_ec_pwm_device { struct cros_ec_device *ec; - struct pwm_chip chip; bool use_pwm_type; struct cros_ec_pwm *channel; }; @@ -40,7 +38,7 @@ struct cros_ec_pwm { static inline struct cros_ec_pwm_device *pwm_to_cros_ec_pwm(struct pwm_chip *chip) { - return container_of(chip, struct cros_ec_pwm_device, chip); + return pwmchip_get_drvdata(chip); } static int cros_ec_dt_type_to_pwm_type(u8 dt_index, u8 *pwm_type) @@ -93,9 +91,8 @@ static int cros_ec_pwm_set_duty(struct cros_ec_pwm_device *ec_pwm, u8 index, return cros_ec_cmd_xfer_status(ec, msg); } -static int cros_ec_pwm_get_duty(struct cros_ec_pwm_device *ec_pwm, u8 index) +static int cros_ec_pwm_get_duty(struct cros_ec_device *ec, bool use_pwm_type, u8 index) { - struct cros_ec_device *ec = ec_pwm->ec; struct { struct cros_ec_command msg; union { @@ -115,7 +112,7 @@ static int cros_ec_pwm_get_duty(struct cros_ec_pwm_device *ec_pwm, u8 index) msg->insize = sizeof(*resp); msg->outsize = sizeof(*params); - if (ec_pwm->use_pwm_type) { + if (use_pwm_type) { ret = cros_ec_dt_type_to_pwm_type(index, ¶ms->pwm_type); if (ret) { dev_err(ec->dev, "Invalid PWM type index: %d\n", index); @@ -171,9 +168,9 @@ static int cros_ec_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, struct cros_ec_pwm *channel = &ec_pwm->channel[pwm->hwpwm]; int ret; - ret = cros_ec_pwm_get_duty(ec_pwm, pwm->hwpwm); + ret = cros_ec_pwm_get_duty(ec_pwm->ec, ec_pwm->use_pwm_type, pwm->hwpwm); if (ret < 0) { - dev_err(chip->dev, "error getting initial duty: %d\n", ret); + dev_err(pwmchip_parent(chip), "error getting initial duty: %d\n", ret); return ret; } @@ -226,13 +223,17 @@ static const struct pwm_ops cros_ec_pwm_ops = { * of PWMs it supports directly, so we have to read the pwm duty cycle for * subsequent channels until we get an error. */ -static int cros_ec_num_pwms(struct cros_ec_pwm_device *ec_pwm) +static int cros_ec_num_pwms(struct cros_ec_device *ec) { int i, ret; /* The index field is only 8 bits */ for (i = 0; i <= U8_MAX; i++) { - ret = cros_ec_pwm_get_duty(ec_pwm, i); + /* + * Note that this function is only called when use_pwm_type is + * false. With use_pwm_type == true the number of PWMs is fixed. + */ + ret = cros_ec_pwm_get_duty(ec, false, i); /* * We look for SUCCESS, INVALID_COMMAND, or INVALID_PARAM * responses; everything else is treated as an error. @@ -261,35 +262,35 @@ static int cros_ec_pwm_probe(struct platform_device *pdev) struct device_node *np = pdev->dev.of_node; struct cros_ec_pwm_device *ec_pwm; struct pwm_chip *chip; + bool use_pwm_type = false; + unsigned int npwm; int ret; if (!ec) return dev_err_probe(dev, -EINVAL, "no parent EC device\n"); - ec_pwm = devm_kzalloc(dev, sizeof(*ec_pwm), GFP_KERNEL); - if (!ec_pwm) - return -ENOMEM; - chip = &ec_pwm->chip; - ec_pwm->ec = ec; - - if (of_device_is_compatible(np, "google,cros-ec-pwm-type")) - ec_pwm->use_pwm_type = true; - - /* PWM chip */ - chip->dev = dev; - chip->ops = &cros_ec_pwm_ops; - chip->of_xlate = cros_ec_pwm_xlate; - chip->of_pwm_n_cells = 1; - - if (ec_pwm->use_pwm_type) { - chip->npwm = CROS_EC_PWM_DT_COUNT; + if (of_device_is_compatible(np, "google,cros-ec-pwm-type")) { + use_pwm_type = true; + npwm = CROS_EC_PWM_DT_COUNT; } else { - ret = cros_ec_num_pwms(ec_pwm); + ret = cros_ec_num_pwms(ec); if (ret < 0) return dev_err_probe(dev, ret, "Couldn't find PWMs\n"); - chip->npwm = ret; + npwm = ret; } + chip = devm_pwmchip_alloc(dev, npwm, sizeof(*ec_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + + ec_pwm = pwm_to_cros_ec_pwm(chip); + ec_pwm->use_pwm_type = use_pwm_type; + ec_pwm->ec = ec; + + /* PWM chip */ + chip->ops = &cros_ec_pwm_ops; + chip->of_xlate = cros_ec_pwm_xlate; + ec_pwm->channel = devm_kcalloc(dev, chip->npwm, sizeof(*ec_pwm->channel), GFP_KERNEL); if (!ec_pwm->channel) diff --git a/drivers/pwm/pwm-dwc-core.c b/drivers/pwm/pwm-dwc-core.c index ea63dd741f5c..043736972cb9 100644 --- a/drivers/pwm/pwm-dwc-core.c +++ b/drivers/pwm/pwm-dwc-core.c @@ -105,12 +105,12 @@ static int dwc_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (state->enabled) { if (!pwm->state.enabled) - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); return __dwc_pwm_configure_timer(dwc, pwm, state); } else { if (pwm->state.enabled) { __dwc_pwm_set_enable(dwc, pwm->hwpwm, false); - pm_runtime_put_sync(chip->dev); + pm_runtime_put_sync(pwmchip_parent(chip)); } } @@ -124,7 +124,7 @@ static int dwc_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, u64 duty, period; u32 ctrl, ld, ld2; - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); ctrl = dwc_pwm_readl(dwc, DWC_TIM_CTRL(pwm->hwpwm)); ld = dwc_pwm_readl(dwc, DWC_TIM_LD_CNT(pwm->hwpwm)); @@ -149,7 +149,7 @@ static int dwc_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, state->period = period; state->duty_cycle = duty; - pm_runtime_put_sync(chip->dev); + pm_runtime_put_sync(pwmchip_parent(chip)); return 0; } @@ -159,21 +159,21 @@ static const struct pwm_ops dwc_pwm_ops = { .get_state = dwc_pwm_get_state, }; -struct dwc_pwm *dwc_pwm_alloc(struct device *dev) +struct pwm_chip *dwc_pwm_alloc(struct device *dev) { + struct pwm_chip *chip; struct dwc_pwm *dwc; - dwc = devm_kzalloc(dev, sizeof(*dwc), GFP_KERNEL); - if (!dwc) - return NULL; + chip = devm_pwmchip_alloc(dev, DWC_TIMERS_TOTAL, sizeof(*dwc)); + if (IS_ERR(chip)) + return chip; + dwc = to_dwc_pwm(chip); dwc->clk_ns = 10; - dwc->chip.dev = dev; - dwc->chip.ops = &dwc_pwm_ops; - dwc->chip.npwm = DWC_TIMERS_TOTAL; + chip->ops = &dwc_pwm_ops; - dev_set_drvdata(dev, dwc); - return dwc; + dev_set_drvdata(dev, chip); + return chip; } EXPORT_SYMBOL_GPL(dwc_pwm_alloc); diff --git a/drivers/pwm/pwm-dwc.c b/drivers/pwm/pwm-dwc.c index 4929354f8cd9..676eaf8d7a53 100644 --- a/drivers/pwm/pwm-dwc.c +++ b/drivers/pwm/pwm-dwc.c @@ -25,39 +25,54 @@ #include "pwm-dwc.h" +/* Elkhart Lake */ +static const struct dwc_pwm_info ehl_pwm_info = { + .nr = 2, + .size = 0x1000, +}; + +static int dwc_pwm_init_one(struct device *dev, void __iomem *base, unsigned int offset) +{ + struct pwm_chip *chip; + struct dwc_pwm *dwc; + + chip = dwc_pwm_alloc(dev); + if (IS_ERR(chip)) + return PTR_ERR(chip); + + dwc = to_dwc_pwm(chip); + dwc->base = base + offset; + + return devm_pwmchip_add(dev, chip); +} + static int dwc_pwm_probe(struct pci_dev *pci, const struct pci_device_id *id) { + const struct dwc_pwm_info *info; struct device *dev = &pci->dev; - struct dwc_pwm *dwc; - int ret; - - dwc = dwc_pwm_alloc(dev); - if (!dwc) - return -ENOMEM; + int i, ret; ret = pcim_enable_device(pci); - if (ret) { - dev_err(dev, "Failed to enable device (%pe)\n", ERR_PTR(ret)); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "Failed to enable device\n"); pci_set_master(pci); ret = pcim_iomap_regions(pci, BIT(0), pci_name(pci)); - if (ret) { - dev_err(dev, "Failed to iomap PCI BAR (%pe)\n", ERR_PTR(ret)); - return ret; - } - - dwc->base = pcim_iomap_table(pci)[0]; - if (!dwc->base) { - dev_err(dev, "Base address missing\n"); - return -ENOMEM; - } - - ret = devm_pwmchip_add(dev, &dwc->chip); if (ret) - return ret; + return dev_err_probe(dev, ret, "Failed to iomap PCI BAR\n"); + + info = (const struct dwc_pwm_info *)id->driver_data; + + for (i = 0; i < info->nr; i++) { + /* + * No need to check for pcim_iomap_table() failure, + * pcim_iomap_regions() already does it for us. + */ + ret = dwc_pwm_init_one(dev, pcim_iomap_table(pci)[0], i * info->size); + if (ret) + return ret; + } pm_runtime_put(dev); pm_runtime_allow(dev); @@ -73,14 +88,14 @@ static void dwc_pwm_remove(struct pci_dev *pci) static int dwc_pwm_suspend(struct device *dev) { - struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); - struct dwc_pwm *dwc = pci_get_drvdata(pdev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct dwc_pwm *dwc = to_dwc_pwm(chip); int i; for (i = 0; i < DWC_TIMERS_TOTAL; i++) { - if (dwc->chip.pwms[i].state.enabled) { + if (chip->pwms[i].state.enabled) { dev_err(dev, "PWM %u in use by consumer (%s)\n", - i, dwc->chip.pwms[i].label); + i, chip->pwms[i].label); return -EBUSY; } dwc->ctx[i].cnt = dwc_pwm_readl(dwc, DWC_TIM_LD_CNT(i)); @@ -93,8 +108,8 @@ static int dwc_pwm_suspend(struct device *dev) static int dwc_pwm_resume(struct device *dev) { - struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); - struct dwc_pwm *dwc = pci_get_drvdata(pdev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct dwc_pwm *dwc = to_dwc_pwm(chip); int i; for (i = 0; i < DWC_TIMERS_TOTAL; i++) { @@ -109,7 +124,7 @@ static int dwc_pwm_resume(struct device *dev) static DEFINE_SIMPLE_DEV_PM_OPS(dwc_pwm_pm_ops, dwc_pwm_suspend, dwc_pwm_resume); static const struct pci_device_id dwc_pwm_id_table[] = { - { PCI_VDEVICE(INTEL, 0x4bb7) }, /* Elkhart Lake */ + { PCI_VDEVICE(INTEL, 0x4bb7), (kernel_ulong_t)&ehl_pwm_info }, { } /* Terminating Entry */ }; MODULE_DEVICE_TABLE(pci, dwc_pwm_id_table); @@ -120,7 +135,7 @@ static struct pci_driver dwc_pwm_driver = { .remove = dwc_pwm_remove, .id_table = dwc_pwm_id_table, .driver = { - .pm = pm_ptr(&dwc_pwm_pm_ops), + .pm = pm_sleep_ptr(&dwc_pwm_pm_ops), }, }; diff --git a/drivers/pwm/pwm-dwc.h b/drivers/pwm/pwm-dwc.h index 64795247c54c..a8b074841ae8 100644 --- a/drivers/pwm/pwm-dwc.h +++ b/drivers/pwm/pwm-dwc.h @@ -33,6 +33,11 @@ MODULE_IMPORT_NS(dwc_pwm); #define DWC_TIM_CTRL_INT_MASK BIT(2) #define DWC_TIM_CTRL_PWM BIT(3) +struct dwc_pwm_info { + unsigned int nr; + unsigned int size; +}; + struct dwc_pwm_ctx { u32 cnt; u32 cnt2; @@ -40,12 +45,15 @@ struct dwc_pwm_ctx { }; struct dwc_pwm { - struct pwm_chip chip; void __iomem *base; unsigned int clk_ns; struct dwc_pwm_ctx ctx[DWC_TIMERS_TOTAL]; }; -#define to_dwc_pwm(p) (container_of((p), struct dwc_pwm, chip)) + +static inline struct dwc_pwm *to_dwc_pwm(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} static inline u32 dwc_pwm_readl(struct dwc_pwm *dwc, u32 offset) { @@ -57,4 +65,4 @@ static inline void dwc_pwm_writel(struct dwc_pwm *dwc, u32 value, u32 offset) writel(value, dwc->base + offset); } -extern struct dwc_pwm *dwc_pwm_alloc(struct device *dev); +extern struct pwm_chip *dwc_pwm_alloc(struct device *dev); diff --git a/drivers/pwm/pwm-ep93xx.c b/drivers/pwm/pwm-ep93xx.c index 51e072572a87..666f2954133c 100644 --- a/drivers/pwm/pwm-ep93xx.c +++ b/drivers/pwm/pwm-ep93xx.c @@ -36,24 +36,23 @@ struct ep93xx_pwm { void __iomem *base; struct clk *clk; - struct pwm_chip chip; }; static inline struct ep93xx_pwm *to_ep93xx_pwm(struct pwm_chip *chip) { - return container_of(chip, struct ep93xx_pwm, chip); + return pwmchip_get_drvdata(chip); } static int ep93xx_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) { - struct platform_device *pdev = to_platform_device(chip->dev); + struct platform_device *pdev = to_platform_device(pwmchip_parent(chip)); return ep93xx_pwm_acquire_gpio(pdev); } static void ep93xx_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) { - struct platform_device *pdev = to_platform_device(chip->dev); + struct platform_device *pdev = to_platform_device(pwmchip_parent(chip)); ep93xx_pwm_release_gpio(pdev); } @@ -163,12 +162,14 @@ static const struct pwm_ops ep93xx_pwm_ops = { static int ep93xx_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct ep93xx_pwm *ep93xx_pwm; int ret; - ep93xx_pwm = devm_kzalloc(&pdev->dev, sizeof(*ep93xx_pwm), GFP_KERNEL); - if (!ep93xx_pwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*ep93xx_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + ep93xx_pwm = to_ep93xx_pwm(chip); ep93xx_pwm->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(ep93xx_pwm->base)) @@ -178,11 +179,9 @@ static int ep93xx_pwm_probe(struct platform_device *pdev) if (IS_ERR(ep93xx_pwm->clk)) return PTR_ERR(ep93xx_pwm->clk); - ep93xx_pwm->chip.dev = &pdev->dev; - ep93xx_pwm->chip.ops = &ep93xx_pwm_ops; - ep93xx_pwm->chip.npwm = 1; + chip->ops = &ep93xx_pwm_ops; - ret = devm_pwmchip_add(&pdev->dev, &ep93xx_pwm->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return ret; diff --git a/drivers/pwm/pwm-fsl-ftm.c b/drivers/pwm/pwm-fsl-ftm.c index d1b6d1aa4773..2510c10ca473 100644 --- a/drivers/pwm/pwm-fsl-ftm.c +++ b/drivers/pwm/pwm-fsl-ftm.c @@ -40,7 +40,6 @@ struct fsl_pwm_periodcfg { }; struct fsl_pwm_chip { - struct pwm_chip chip; struct mutex lock; struct regmap *regmap; @@ -55,7 +54,7 @@ struct fsl_pwm_chip { static inline struct fsl_pwm_chip *to_fsl_chip(struct pwm_chip *chip) { - return container_of(chip, struct fsl_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static void ftm_clear_write_protection(struct fsl_pwm_chip *fpc) @@ -221,10 +220,11 @@ static bool fsl_pwm_is_other_pwm_enabled(struct fsl_pwm_chip *fpc, return false; } -static int fsl_pwm_apply_config(struct fsl_pwm_chip *fpc, +static int fsl_pwm_apply_config(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *newstate) { + struct fsl_pwm_chip *fpc = to_fsl_chip(chip); unsigned int duty; u32 reg_polarity; @@ -232,7 +232,7 @@ static int fsl_pwm_apply_config(struct fsl_pwm_chip *fpc, bool do_write_period = false; if (!fsl_pwm_calculate_period(fpc, newstate->period, &periodcfg)) { - dev_err(fpc->chip.dev, "failed to calculate new period\n"); + dev_err(pwmchip_parent(chip), "failed to calculate new period\n"); return -EINVAL; } @@ -246,7 +246,7 @@ static int fsl_pwm_apply_config(struct fsl_pwm_chip *fpc, */ else if (!fsl_pwm_periodcfg_are_equal(&fpc->period, &periodcfg)) { if (fsl_pwm_is_other_pwm_enabled(fpc, pwm)) { - dev_err(fpc->chip.dev, + dev_err(pwmchip_parent(chip), "Cannot change period for PWM %u, disable other PWMs first\n", pwm->hwpwm); return -EBUSY; @@ -322,7 +322,7 @@ static int fsl_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, goto end_mutex; } - ret = fsl_pwm_apply_config(fpc, pwm, newstate); + ret = fsl_pwm_apply_config(chip, pwm, newstate); if (ret) goto end_mutex; @@ -392,18 +392,19 @@ static const struct regmap_config fsl_pwm_regmap_config = { static int fsl_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct fsl_pwm_chip *fpc; void __iomem *base; int ret; - fpc = devm_kzalloc(&pdev->dev, sizeof(*fpc), GFP_KERNEL); - if (!fpc) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 8, sizeof(*fpc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + fpc = to_fsl_chip(chip); mutex_init(&fpc->lock); fpc->soc = of_device_get_match_data(&pdev->dev); - fpc->chip.dev = &pdev->dev; base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base)) @@ -422,16 +423,16 @@ static int fsl_pwm_probe(struct platform_device *pdev) return PTR_ERR(fpc->clk[FSL_PWM_CLK_SYS]); } - fpc->clk[FSL_PWM_CLK_FIX] = devm_clk_get(fpc->chip.dev, "ftm_fix"); + fpc->clk[FSL_PWM_CLK_FIX] = devm_clk_get(&pdev->dev, "ftm_fix"); if (IS_ERR(fpc->clk[FSL_PWM_CLK_FIX])) return PTR_ERR(fpc->clk[FSL_PWM_CLK_FIX]); - fpc->clk[FSL_PWM_CLK_EXT] = devm_clk_get(fpc->chip.dev, "ftm_ext"); + fpc->clk[FSL_PWM_CLK_EXT] = devm_clk_get(&pdev->dev, "ftm_ext"); if (IS_ERR(fpc->clk[FSL_PWM_CLK_EXT])) return PTR_ERR(fpc->clk[FSL_PWM_CLK_EXT]); fpc->clk[FSL_PWM_CLK_CNTEN] = - devm_clk_get(fpc->chip.dev, "ftm_cnt_clk_en"); + devm_clk_get(&pdev->dev, "ftm_cnt_clk_en"); if (IS_ERR(fpc->clk[FSL_PWM_CLK_CNTEN])) return PTR_ERR(fpc->clk[FSL_PWM_CLK_CNTEN]); @@ -443,17 +444,15 @@ static int fsl_pwm_probe(struct platform_device *pdev) if (IS_ERR(fpc->ipg_clk)) fpc->ipg_clk = fpc->clk[FSL_PWM_CLK_SYS]; + chip->ops = &fsl_pwm_ops; - fpc->chip.ops = &fsl_pwm_ops; - fpc->chip.npwm = 8; - - ret = devm_pwmchip_add(&pdev->dev, &fpc->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) { dev_err(&pdev->dev, "failed to add PWM chip: %d\n", ret); return ret; } - platform_set_drvdata(pdev, fpc); + platform_set_drvdata(pdev, chip); return fsl_pwm_init(fpc); } @@ -461,14 +460,15 @@ static int fsl_pwm_probe(struct platform_device *pdev) #ifdef CONFIG_PM_SLEEP static int fsl_pwm_suspend(struct device *dev) { - struct fsl_pwm_chip *fpc = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct fsl_pwm_chip *fpc = to_fsl_chip(chip); int i; regcache_cache_only(fpc->regmap, true); regcache_mark_dirty(fpc->regmap); - for (i = 0; i < fpc->chip.npwm; i++) { - struct pwm_device *pwm = &fpc->chip.pwms[i]; + for (i = 0; i < chip->npwm; i++) { + struct pwm_device *pwm = &chip->pwms[i]; if (!test_bit(PWMF_REQUESTED, &pwm->flags)) continue; @@ -487,11 +487,12 @@ static int fsl_pwm_suspend(struct device *dev) static int fsl_pwm_resume(struct device *dev) { - struct fsl_pwm_chip *fpc = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct fsl_pwm_chip *fpc = to_fsl_chip(chip); int i; - for (i = 0; i < fpc->chip.npwm; i++) { - struct pwm_device *pwm = &fpc->chip.pwms[i]; + for (i = 0; i < chip->npwm; i++) { + struct pwm_device *pwm = &chip->pwms[i]; if (!test_bit(PWMF_REQUESTED, &pwm->flags)) continue; diff --git a/drivers/pwm/pwm-hibvt.c b/drivers/pwm/pwm-hibvt.c index c435776e2f78..2eb0b13d4e10 100644 --- a/drivers/pwm/pwm-hibvt.c +++ b/drivers/pwm/pwm-hibvt.c @@ -33,7 +33,6 @@ #define PWM_DUTY_MASK GENMASK(31, 0) struct hibvt_pwm_chip { - struct pwm_chip chip; struct clk *clk; void __iomem *base; struct reset_control *rstc; @@ -65,7 +64,7 @@ static const struct hibvt_pwm_soc hi3559v100_soc_info = { static inline struct hibvt_pwm_chip *to_hibvt_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct hibvt_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static void hibvt_pwm_set_bits(void __iomem *base, u32 offset, @@ -191,72 +190,71 @@ static int hibvt_pwm_probe(struct platform_device *pdev) { const struct hibvt_pwm_soc *soc = of_device_get_match_data(&pdev->dev); - struct hibvt_pwm_chip *pwm_chip; + struct pwm_chip *chip; + struct hibvt_pwm_chip *hi_pwm_chip; int ret, i; - pwm_chip = devm_kzalloc(&pdev->dev, sizeof(*pwm_chip), GFP_KERNEL); - if (pwm_chip == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, soc->num_pwms, sizeof(*hi_pwm_chip)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + hi_pwm_chip = to_hibvt_pwm_chip(chip); - pwm_chip->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(pwm_chip->clk)) { + hi_pwm_chip->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(hi_pwm_chip->clk)) { dev_err(&pdev->dev, "getting clock failed with %ld\n", - PTR_ERR(pwm_chip->clk)); - return PTR_ERR(pwm_chip->clk); + PTR_ERR(hi_pwm_chip->clk)); + return PTR_ERR(hi_pwm_chip->clk); } - pwm_chip->chip.ops = &hibvt_pwm_ops; - pwm_chip->chip.dev = &pdev->dev; - pwm_chip->chip.npwm = soc->num_pwms; - pwm_chip->soc = soc; + chip->ops = &hibvt_pwm_ops; + hi_pwm_chip->soc = soc; - pwm_chip->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(pwm_chip->base)) - return PTR_ERR(pwm_chip->base); + hi_pwm_chip->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(hi_pwm_chip->base)) + return PTR_ERR(hi_pwm_chip->base); - ret = clk_prepare_enable(pwm_chip->clk); + ret = clk_prepare_enable(hi_pwm_chip->clk); if (ret < 0) return ret; - pwm_chip->rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL); - if (IS_ERR(pwm_chip->rstc)) { - clk_disable_unprepare(pwm_chip->clk); - return PTR_ERR(pwm_chip->rstc); + hi_pwm_chip->rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL); + if (IS_ERR(hi_pwm_chip->rstc)) { + clk_disable_unprepare(hi_pwm_chip->clk); + return PTR_ERR(hi_pwm_chip->rstc); } - reset_control_assert(pwm_chip->rstc); + reset_control_assert(hi_pwm_chip->rstc); msleep(30); - reset_control_deassert(pwm_chip->rstc); + reset_control_deassert(hi_pwm_chip->rstc); - ret = pwmchip_add(&pwm_chip->chip); + ret = pwmchip_add(chip); if (ret < 0) { - clk_disable_unprepare(pwm_chip->clk); + clk_disable_unprepare(hi_pwm_chip->clk); return ret; } - for (i = 0; i < pwm_chip->chip.npwm; i++) { - hibvt_pwm_set_bits(pwm_chip->base, PWM_CTRL_ADDR(i), + for (i = 0; i < chip->npwm; i++) { + hibvt_pwm_set_bits(hi_pwm_chip->base, PWM_CTRL_ADDR(i), PWM_KEEP_MASK, (0x1 << PWM_KEEP_SHIFT)); } - platform_set_drvdata(pdev, pwm_chip); + platform_set_drvdata(pdev, chip); return 0; } static void hibvt_pwm_remove(struct platform_device *pdev) { - struct hibvt_pwm_chip *pwm_chip; + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct hibvt_pwm_chip *hi_pwm_chip = to_hibvt_pwm_chip(chip); - pwm_chip = platform_get_drvdata(pdev); + pwmchip_remove(chip); - pwmchip_remove(&pwm_chip->chip); - - reset_control_assert(pwm_chip->rstc); + reset_control_assert(hi_pwm_chip->rstc); msleep(30); - reset_control_deassert(pwm_chip->rstc); + reset_control_deassert(hi_pwm_chip->rstc); - clk_disable_unprepare(pwm_chip->clk); + clk_disable_unprepare(hi_pwm_chip->clk); } static const struct of_device_id hibvt_pwm_of_match[] = { diff --git a/drivers/pwm/pwm-img.c b/drivers/pwm/pwm-img.c index 5965ac35b32e..d79a96679a26 100644 --- a/drivers/pwm/pwm-img.c +++ b/drivers/pwm/pwm-img.c @@ -59,8 +59,6 @@ struct img_pwm_soc_data { }; struct img_pwm_chip { - struct device *dev; - struct pwm_chip chip; struct clk *pwm_clk; struct clk *sys_clk; void __iomem *base; @@ -74,7 +72,7 @@ struct img_pwm_chip { static inline struct img_pwm_chip *to_img_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct img_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static inline void img_pwm_writel(struct img_pwm_chip *imgchip, @@ -99,7 +97,7 @@ static int img_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if (period_ns < imgchip->min_period_ns || period_ns > imgchip->max_period_ns) { - dev_err(chip->dev, "configured period not in range\n"); + dev_err(pwmchip_parent(chip), "configured period not in range\n"); return -ERANGE; } @@ -120,14 +118,14 @@ static int img_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, div = PWM_CTRL_CFG_SUB_DIV0_DIV1; timebase = DIV_ROUND_UP(mul, 512); } else { - dev_err(chip->dev, + dev_err(pwmchip_parent(chip), "failed to configure timebase steps/divider value\n"); return -EINVAL; } duty = DIV_ROUND_UP(timebase * duty_ns, period_ns); - ret = pm_runtime_resume_and_get(chip->dev); + ret = pm_runtime_resume_and_get(pwmchip_parent(chip)); if (ret < 0) return ret; @@ -141,8 +139,8 @@ static int img_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, (timebase << PWM_CH_CFG_TMBASE_SHIFT); img_pwm_writel(imgchip, PWM_CH_CFG(pwm->hwpwm), val); - pm_runtime_mark_last_busy(chip->dev); - pm_runtime_put_autosuspend(chip->dev); + pm_runtime_mark_last_busy(pwmchip_parent(chip)); + pm_runtime_put_autosuspend(pwmchip_parent(chip)); return 0; } @@ -153,7 +151,7 @@ static int img_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) struct img_pwm_chip *imgchip = to_img_pwm_chip(chip); int ret; - ret = pm_runtime_resume_and_get(chip->dev); + ret = pm_runtime_resume_and_get(pwmchip_parent(chip)); if (ret < 0) return ret; @@ -177,8 +175,8 @@ static void img_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) val &= ~BIT(pwm->hwpwm); img_pwm_writel(imgchip, PWM_CTRL_CFG, val); - pm_runtime_mark_last_busy(chip->dev); - pm_runtime_put_autosuspend(chip->dev); + pm_runtime_mark_last_busy(pwmchip_parent(chip)); + pm_runtime_put_autosuspend(pwmchip_parent(chip)); } static int img_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, @@ -225,7 +223,8 @@ MODULE_DEVICE_TABLE(of, img_pwm_of_match); static int img_pwm_runtime_suspend(struct device *dev) { - struct img_pwm_chip *imgchip = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct img_pwm_chip *imgchip = to_img_pwm_chip(chip); clk_disable_unprepare(imgchip->pwm_clk); clk_disable_unprepare(imgchip->sys_clk); @@ -235,7 +234,8 @@ static int img_pwm_runtime_suspend(struct device *dev) static int img_pwm_runtime_resume(struct device *dev) { - struct img_pwm_chip *imgchip = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct img_pwm_chip *imgchip = to_img_pwm_chip(chip); int ret; ret = clk_prepare_enable(imgchip->sys_clk); @@ -259,13 +259,13 @@ static int img_pwm_probe(struct platform_device *pdev) int ret; u64 val; unsigned long clk_rate; + struct pwm_chip *chip; struct img_pwm_chip *imgchip; - imgchip = devm_kzalloc(&pdev->dev, sizeof(*imgchip), GFP_KERNEL); - if (!imgchip) - return -ENOMEM; - - imgchip->dev = &pdev->dev; + chip = devm_pwmchip_alloc(&pdev->dev, IMG_PWM_NPWM, sizeof(*imgchip)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + imgchip = to_img_pwm_chip(chip); imgchip->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(imgchip->base)) @@ -290,7 +290,7 @@ static int img_pwm_probe(struct platform_device *pdev) return PTR_ERR(imgchip->pwm_clk); } - platform_set_drvdata(pdev, imgchip); + platform_set_drvdata(pdev, chip); pm_runtime_set_autosuspend_delay(&pdev->dev, IMG_PWM_PM_TIMEOUT); pm_runtime_use_autosuspend(&pdev->dev); @@ -317,11 +317,9 @@ static int img_pwm_probe(struct platform_device *pdev) do_div(val, clk_rate); imgchip->min_period_ns = val; - imgchip->chip.dev = &pdev->dev; - imgchip->chip.ops = &img_pwm_ops; - imgchip->chip.npwm = IMG_PWM_NPWM; + chip->ops = &img_pwm_ops; - ret = pwmchip_add(&imgchip->chip); + ret = pwmchip_add(chip); if (ret < 0) { dev_err(&pdev->dev, "pwmchip_add failed: %d\n", ret); goto err_suspend; @@ -340,19 +338,20 @@ err_pm_disable: static void img_pwm_remove(struct platform_device *pdev) { - struct img_pwm_chip *imgchip = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); pm_runtime_disable(&pdev->dev); if (!pm_runtime_status_suspended(&pdev->dev)) img_pwm_runtime_suspend(&pdev->dev); - pwmchip_remove(&imgchip->chip); + pwmchip_remove(chip); } #ifdef CONFIG_PM_SLEEP static int img_pwm_suspend(struct device *dev) { - struct img_pwm_chip *imgchip = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct img_pwm_chip *imgchip = to_img_pwm_chip(chip); int i, ret; if (pm_runtime_status_suspended(dev)) { @@ -361,7 +360,7 @@ static int img_pwm_suspend(struct device *dev) return ret; } - for (i = 0; i < imgchip->chip.npwm; i++) + for (i = 0; i < chip->npwm; i++) imgchip->suspend_ch_cfg[i] = img_pwm_readl(imgchip, PWM_CH_CFG(i)); @@ -374,7 +373,8 @@ static int img_pwm_suspend(struct device *dev) static int img_pwm_resume(struct device *dev) { - struct img_pwm_chip *imgchip = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct img_pwm_chip *imgchip = to_img_pwm_chip(chip); int ret; int i; @@ -382,13 +382,13 @@ static int img_pwm_resume(struct device *dev) if (ret) return ret; - for (i = 0; i < imgchip->chip.npwm; i++) + for (i = 0; i < chip->npwm; i++) img_pwm_writel(imgchip, PWM_CH_CFG(i), imgchip->suspend_ch_cfg[i]); img_pwm_writel(imgchip, PWM_CTRL_CFG, imgchip->suspend_ctrl_cfg); - for (i = 0; i < imgchip->chip.npwm; i++) + for (i = 0; i < chip->npwm; i++) if (imgchip->suspend_ctrl_cfg & BIT(i)) regmap_clear_bits(imgchip->periph_regs, PERIP_PWM_PDM_CONTROL, diff --git a/drivers/pwm/pwm-imx-tpm.c b/drivers/pwm/pwm-imx-tpm.c index 9fc290e647e1..c50ddbac43c8 100644 --- a/drivers/pwm/pwm-imx-tpm.c +++ b/drivers/pwm/pwm-imx-tpm.c @@ -57,7 +57,6 @@ #define PWM_IMX_TPM_MOD_MOD GENMASK(PWM_IMX_TPM_MOD_WIDTH - 1, 0) struct imx_tpm_pwm_chip { - struct pwm_chip chip; struct clk *clk; void __iomem *base; struct mutex lock; @@ -75,7 +74,7 @@ struct imx_tpm_pwm_param { static inline struct imx_tpm_pwm_chip * to_imx_tpm_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct imx_tpm_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } /* @@ -336,35 +335,42 @@ static const struct pwm_ops imx_tpm_pwm_ops = { static int pwm_imx_tpm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct imx_tpm_pwm_chip *tpm; + struct clk *clk; + void __iomem *base; int ret; + unsigned int npwm; u32 val; - tpm = devm_kzalloc(&pdev->dev, sizeof(*tpm), GFP_KERNEL); - if (!tpm) - return -ENOMEM; + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + clk = devm_clk_get_enabled(&pdev->dev, NULL); + if (IS_ERR(clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(clk), + "failed to get PWM clock\n"); + + /* get number of channels */ + val = readl(base + PWM_IMX_TPM_PARAM); + npwm = FIELD_GET(PWM_IMX_TPM_PARAM_CHAN, val); + + chip = devm_pwmchip_alloc(&pdev->dev, npwm, sizeof(*tpm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + tpm = to_imx_tpm_pwm_chip(chip); platform_set_drvdata(pdev, tpm); - tpm->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(tpm->base)) - return PTR_ERR(tpm->base); + tpm->base = base; + tpm->clk = clk; - tpm->clk = devm_clk_get_enabled(&pdev->dev, NULL); - if (IS_ERR(tpm->clk)) - return dev_err_probe(&pdev->dev, PTR_ERR(tpm->clk), - "failed to get PWM clock\n"); - - tpm->chip.dev = &pdev->dev; - tpm->chip.ops = &imx_tpm_pwm_ops; - - /* get number of channels */ - val = readl(tpm->base + PWM_IMX_TPM_PARAM); - tpm->chip.npwm = FIELD_GET(PWM_IMX_TPM_PARAM_CHAN, val); + chip->ops = &imx_tpm_pwm_ops; mutex_init(&tpm->lock); - ret = devm_pwmchip_add(&pdev->dev, &tpm->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret) return dev_err_probe(&pdev->dev, ret, "failed to add PWM chip\n"); diff --git a/drivers/pwm/pwm-imx1.c b/drivers/pwm/pwm-imx1.c index d175d895f22a..1d2aae2d278f 100644 --- a/drivers/pwm/pwm-imx1.c +++ b/drivers/pwm/pwm-imx1.c @@ -28,10 +28,12 @@ struct pwm_imx1_chip { struct clk *clk_ipg; struct clk *clk_per; void __iomem *mmio_base; - struct pwm_chip chip; }; -#define to_pwm_imx1_chip(chip) container_of(chip, struct pwm_imx1_chip, chip) +static inline struct pwm_imx1_chip *to_pwm_imx1_chip(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} static int pwm_imx1_clk_prepare_enable(struct pwm_chip *chip) { @@ -156,11 +158,13 @@ MODULE_DEVICE_TABLE(of, pwm_imx1_dt_ids); static int pwm_imx1_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct pwm_imx1_chip *imx; - imx = devm_kzalloc(&pdev->dev, sizeof(*imx), GFP_KERNEL); - if (!imx) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*imx)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + imx = to_pwm_imx1_chip(chip); imx->clk_ipg = devm_clk_get(&pdev->dev, "ipg"); if (IS_ERR(imx->clk_ipg)) @@ -172,15 +176,13 @@ static int pwm_imx1_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(imx->clk_per), "failed to get peripheral clock\n"); - imx->chip.ops = &pwm_imx1_ops; - imx->chip.dev = &pdev->dev; - imx->chip.npwm = 1; + chip->ops = &pwm_imx1_ops; imx->mmio_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(imx->mmio_base)) return PTR_ERR(imx->mmio_base); - return devm_pwmchip_add(&pdev->dev, &imx->chip); + return devm_pwmchip_add(&pdev->dev, chip); } static struct platform_driver pwm_imx1_driver = { diff --git a/drivers/pwm/pwm-imx27.c b/drivers/pwm/pwm-imx27.c index 7d9bc43f12b0..e1412116ef65 100644 --- a/drivers/pwm/pwm-imx27.c +++ b/drivers/pwm/pwm-imx27.c @@ -83,7 +83,6 @@ struct pwm_imx27_chip { struct clk *clk_ipg; struct clk *clk_per; void __iomem *mmio_base; - struct pwm_chip chip; /* * The driver cannot read the current duty cycle from the hardware if @@ -93,7 +92,10 @@ struct pwm_imx27_chip { unsigned int duty_cycle; }; -#define to_pwm_imx27_chip(chip) container_of(chip, struct pwm_imx27_chip, chip) +static inline struct pwm_imx27_chip *to_pwm_imx27_chip(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} static int pwm_imx27_clk_prepare_enable(struct pwm_imx27_chip *imx) { @@ -145,7 +147,7 @@ static int pwm_imx27_get_state(struct pwm_chip *chip, state->polarity = PWM_POLARITY_INVERSED; break; default: - dev_warn(chip->dev, "can't set polarity, output disconnected"); + dev_warn(pwmchip_parent(chip), "can't set polarity, output disconnected"); } prescaler = MX3_PWMCR_PRESCALER_GET(val); @@ -177,7 +179,7 @@ static int pwm_imx27_get_state(struct pwm_chip *chip, static void pwm_imx27_sw_reset(struct pwm_chip *chip) { struct pwm_imx27_chip *imx = to_pwm_imx27_chip(chip); - struct device *dev = chip->dev; + struct device *dev = pwmchip_parent(chip); int wait_count = 0; u32 cr; @@ -196,7 +198,7 @@ static void pwm_imx27_wait_fifo_slot(struct pwm_chip *chip, struct pwm_device *pwm) { struct pwm_imx27_chip *imx = to_pwm_imx27_chip(chip); - struct device *dev = chip->dev; + struct device *dev = pwmchip_parent(chip); unsigned int period_ms; int fifoav; u32 sr; @@ -204,8 +206,8 @@ static void pwm_imx27_wait_fifo_slot(struct pwm_chip *chip, sr = readl(imx->mmio_base + MX3_PWMSR); fifoav = FIELD_GET(MX3_PWMSR_FIFOAV, sr); if (fifoav == MX3_PWMSR_FIFOAV_4WORDS) { - period_ms = DIV_ROUND_UP_ULL(pwm_get_period(pwm), - NSEC_PER_MSEC); + period_ms = DIV_ROUND_UP_ULL(pwm->state.period, + NSEC_PER_MSEC); msleep(period_ms); sr = readl(imx->mmio_base + MX3_PWMSR); @@ -219,14 +221,11 @@ static int pwm_imx27_apply(struct pwm_chip *chip, struct pwm_device *pwm, { unsigned long period_cycles, duty_cycles, prescale; struct pwm_imx27_chip *imx = to_pwm_imx27_chip(chip); - struct pwm_state cstate; unsigned long long c; unsigned long long clkrate; int ret; u32 cr; - pwm_get_state(pwm, &cstate); - clkrate = clk_get_rate(imx->clk_per); c = clkrate * state->period; @@ -254,7 +253,7 @@ static int pwm_imx27_apply(struct pwm_chip *chip, struct pwm_device *pwm, * Wait for a free FIFO slot if the PWM is already enabled, and flush * the FIFO if the PWM was disabled and is about to be enabled. */ - if (cstate.enabled) { + if (pwm->state.enabled) { pwm_imx27_wait_fifo_slot(chip, pwm); } else { ret = pwm_imx27_clk_prepare_enable(imx); @@ -306,13 +305,15 @@ MODULE_DEVICE_TABLE(of, pwm_imx27_dt_ids); static int pwm_imx27_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct pwm_imx27_chip *imx; int ret; u32 pwmcr; - imx = devm_kzalloc(&pdev->dev, sizeof(*imx), GFP_KERNEL); - if (imx == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*imx)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + imx = to_pwm_imx27_chip(chip); imx->clk_ipg = devm_clk_get(&pdev->dev, "ipg"); if (IS_ERR(imx->clk_ipg)) @@ -324,9 +325,7 @@ static int pwm_imx27_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(imx->clk_per), "failed to get peripheral clock\n"); - imx->chip.ops = &pwm_imx27_ops; - imx->chip.dev = &pdev->dev; - imx->chip.npwm = 1; + chip->ops = &pwm_imx27_ops; imx->mmio_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(imx->mmio_base)) @@ -341,7 +340,7 @@ static int pwm_imx27_probe(struct platform_device *pdev) if (!(pwmcr & MX3_PWMCR_EN)) pwm_imx27_clk_disable_unprepare(imx); - return devm_pwmchip_add(&pdev->dev, &imx->chip); + return devm_pwmchip_add(&pdev->dev, chip); } static struct platform_driver imx_pwm_driver = { diff --git a/drivers/pwm/pwm-intel-lgm.c b/drivers/pwm/pwm-intel-lgm.c index 54ecae7f937e..f9cc7c17c8f0 100644 --- a/drivers/pwm/pwm-intel-lgm.c +++ b/drivers/pwm/pwm-intel-lgm.c @@ -42,14 +42,13 @@ #define LGM_PWM_PERIOD_2WIRE_NS (40 * NSEC_PER_MSEC) struct lgm_pwm_chip { - struct pwm_chip chip; struct regmap *regmap; u32 period; }; static inline struct lgm_pwm_chip *to_lgm_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct lgm_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static int lgm_pwm_enable(struct pwm_chip *chip, bool enable) @@ -168,14 +167,16 @@ static int lgm_pwm_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct reset_control *rst; + struct pwm_chip *chip; struct lgm_pwm_chip *pc; void __iomem *io_base; struct clk *clk; int ret; - pc = devm_kzalloc(dev, sizeof(*pc), GFP_KERNEL); - if (!pc) - return -ENOMEM; + chip = devm_pwmchip_alloc(dev, 1, sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_lgm_pwm_chip(chip); io_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(io_base)) @@ -203,13 +204,11 @@ static int lgm_pwm_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "cannot deassert reset control\n"); - pc->chip.dev = dev; - pc->chip.ops = &lgm_pwm_ops; - pc->chip.npwm = 1; + chip->ops = &lgm_pwm_ops; lgm_pwm_init(pc); - ret = devm_pwmchip_add(dev, &pc->chip); + ret = devm_pwmchip_add(dev, chip); if (ret < 0) return dev_err_probe(dev, ret, "failed to add PWM chip\n"); diff --git a/drivers/pwm/pwm-iqs620a.c b/drivers/pwm/pwm-iqs620a.c index 378ab036edfe..13e5e138c8e9 100644 --- a/drivers/pwm/pwm-iqs620a.c +++ b/drivers/pwm/pwm-iqs620a.c @@ -34,12 +34,17 @@ struct iqs620_pwm_private { struct iqs62x_core *iqs62x; - struct pwm_chip chip; + struct device *dev; struct notifier_block notifier; struct mutex lock; unsigned int duty_scale; }; +static inline struct iqs620_pwm_private *iqs620_pwm_from_chip(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} + static int iqs620_pwm_init(struct iqs620_pwm_private *iqs620_pwm, unsigned int duty_scale) { @@ -73,7 +78,7 @@ static int iqs620_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (state->period < IQS620_PWM_PERIOD_NS) return -EINVAL; - iqs620_pwm = container_of(chip, struct iqs620_pwm_private, chip); + iqs620_pwm = iqs620_pwm_from_chip(chip); /* * The duty cycle generated by the device is calculated as follows: @@ -109,7 +114,7 @@ static int iqs620_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, { struct iqs620_pwm_private *iqs620_pwm; - iqs620_pwm = container_of(chip, struct iqs620_pwm_private, chip); + iqs620_pwm = iqs620_pwm_from_chip(chip); mutex_lock(&iqs620_pwm->lock); @@ -155,7 +160,7 @@ static int iqs620_pwm_notifier(struct notifier_block *notifier, mutex_unlock(&iqs620_pwm->lock); if (ret) { - dev_err(iqs620_pwm->chip.dev, + dev_err(iqs620_pwm->dev, "Failed to re-initialize device: %d\n", ret); return NOTIFY_BAD; } @@ -176,21 +181,24 @@ static void iqs620_pwm_notifier_unregister(void *context) ret = blocking_notifier_chain_unregister(&iqs620_pwm->iqs62x->nh, &iqs620_pwm->notifier); if (ret) - dev_err(iqs620_pwm->chip.dev, + dev_err(iqs620_pwm->dev, "Failed to unregister notifier: %d\n", ret); } static int iqs620_pwm_probe(struct platform_device *pdev) { struct iqs62x_core *iqs62x = dev_get_drvdata(pdev->dev.parent); + struct pwm_chip *chip; struct iqs620_pwm_private *iqs620_pwm; unsigned int val; int ret; - iqs620_pwm = devm_kzalloc(&pdev->dev, sizeof(*iqs620_pwm), GFP_KERNEL); - if (!iqs620_pwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*iqs620_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + iqs620_pwm = iqs620_pwm_from_chip(chip); + iqs620_pwm->dev = &pdev->dev; iqs620_pwm->iqs62x = iqs62x; ret = regmap_read(iqs62x->regmap, IQS620_PWR_SETTINGS, &val); @@ -205,9 +213,7 @@ static int iqs620_pwm_probe(struct platform_device *pdev) iqs620_pwm->duty_scale = val + 1; } - iqs620_pwm->chip.dev = &pdev->dev; - iqs620_pwm->chip.ops = &iqs620_pwm_ops; - iqs620_pwm->chip.npwm = 1; + chip->ops = &iqs620_pwm_ops; mutex_init(&iqs620_pwm->lock); @@ -225,7 +231,7 @@ static int iqs620_pwm_probe(struct platform_device *pdev) if (ret) return ret; - ret = devm_pwmchip_add(&pdev->dev, &iqs620_pwm->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret) dev_err(&pdev->dev, "Failed to add device: %d\n", ret); diff --git a/drivers/pwm/pwm-jz4740.c b/drivers/pwm/pwm-jz4740.c index 3933418e551b..da4bf543d357 100644 --- a/drivers/pwm/pwm-jz4740.c +++ b/drivers/pwm/pwm-jz4740.c @@ -25,23 +25,21 @@ struct soc_info { }; struct jz4740_pwm_chip { - struct pwm_chip chip; struct regmap *map; struct clk *clk[]; }; static inline struct jz4740_pwm_chip *to_jz4740(struct pwm_chip *chip) { - return container_of(chip, struct jz4740_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } -static bool jz4740_pwm_can_use_chn(struct jz4740_pwm_chip *jz, - unsigned int channel) +static bool jz4740_pwm_can_use_chn(struct pwm_chip *chip, unsigned int channel) { /* Enable all TCU channels for PWM use by default except channels 0/1 */ - u32 pwm_channels_mask = GENMASK(jz->chip.npwm - 1, 2); + u32 pwm_channels_mask = GENMASK(chip->npwm - 1, 2); - device_property_read_u32(jz->chip.dev->parent, + device_property_read_u32(pwmchip_parent(chip)->parent, "ingenic,pwm-channels-mask", &pwm_channels_mask); @@ -55,14 +53,15 @@ static int jz4740_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) char name[16]; int err; - if (!jz4740_pwm_can_use_chn(jz, pwm->hwpwm)) + if (!jz4740_pwm_can_use_chn(chip, pwm->hwpwm)) return -EBUSY; snprintf(name, sizeof(name), "timer%u", pwm->hwpwm); - clk = clk_get(chip->dev, name); + clk = clk_get(pwmchip_parent(chip), name); if (IS_ERR(clk)) { - dev_err(chip->dev, "error %pe: Failed to get clock\n", clk); + dev_err(pwmchip_parent(chip), + "error %pe: Failed to get clock\n", clk); return PTR_ERR(clk); } @@ -150,7 +149,7 @@ static int jz4740_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, */ rate = clk_round_rate(clk, tmp); if (rate < 0) { - dev_err(chip->dev, "Unable to round rate: %ld\n", rate); + dev_err(pwmchip_parent(chip), "Unable to round rate: %ld\n", rate); return rate; } @@ -171,7 +170,7 @@ static int jz4740_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, err = clk_set_rate(clk, rate); if (err) { - dev_err(chip->dev, "Unable to set rate: %d\n", err); + dev_err(pwmchip_parent(chip), "Unable to set rate: %d\n", err); return err; } @@ -224,6 +223,7 @@ static const struct pwm_ops jz4740_pwm_ops = { static int jz4740_pwm_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + struct pwm_chip *chip; struct jz4740_pwm_chip *jz; const struct soc_info *info; @@ -231,10 +231,10 @@ static int jz4740_pwm_probe(struct platform_device *pdev) if (!info) return -EINVAL; - jz = devm_kzalloc(dev, struct_size(jz, clk, info->num_pwms), - GFP_KERNEL); - if (!jz) - return -ENOMEM; + chip = devm_pwmchip_alloc(dev, info->num_pwms, struct_size(jz, clk, info->num_pwms)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + jz = to_jz4740(chip); jz->map = device_node_to_regmap(dev->parent->of_node); if (IS_ERR(jz->map)) { @@ -242,11 +242,9 @@ static int jz4740_pwm_probe(struct platform_device *pdev) return PTR_ERR(jz->map); } - jz->chip.dev = dev; - jz->chip.ops = &jz4740_pwm_ops; - jz->chip.npwm = info->num_pwms; + chip->ops = &jz4740_pwm_ops; - return devm_pwmchip_add(dev, &jz->chip); + return devm_pwmchip_add(dev, chip); } static const struct soc_info jz4740_soc_info = { diff --git a/drivers/pwm/pwm-keembay.c b/drivers/pwm/pwm-keembay.c index ac824ecc3f64..35b641f3f6ed 100644 --- a/drivers/pwm/pwm-keembay.c +++ b/drivers/pwm/pwm-keembay.c @@ -36,7 +36,6 @@ #define KMB_PWM_HIGHLOW_OFFSET(ch) (0x20 + 4 * (ch)) struct keembay_pwm { - struct pwm_chip chip; struct device *dev; struct clk *clk; void __iomem *base; @@ -44,7 +43,7 @@ struct keembay_pwm { static inline struct keembay_pwm *to_keembay_pwm_dev(struct pwm_chip *chip) { - return container_of(chip, struct keembay_pwm, chip); + return pwmchip_get_drvdata(chip); } static void keembay_clk_unprepare(void *data) @@ -185,12 +184,14 @@ static const struct pwm_ops keembay_pwm_ops = { static int keembay_pwm_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + struct pwm_chip *chip; struct keembay_pwm *priv; int ret; - priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; + chip = devm_pwmchip_alloc(dev, KMB_TOTAL_PWM_CHANNELS, sizeof(*priv)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + priv = to_keembay_pwm_dev(chip); priv->clk = devm_clk_get(dev, NULL); if (IS_ERR(priv->clk)) @@ -204,11 +205,9 @@ static int keembay_pwm_probe(struct platform_device *pdev) if (ret) return ret; - priv->chip.dev = dev; - priv->chip.ops = &keembay_pwm_ops; - priv->chip.npwm = KMB_TOTAL_PWM_CHANNELS; + chip->ops = &keembay_pwm_ops; - ret = devm_pwmchip_add(dev, &priv->chip); + ret = devm_pwmchip_add(dev, chip); if (ret) return dev_err_probe(dev, ret, "Failed to add PWM chip\n"); diff --git a/drivers/pwm/pwm-lp3943.c b/drivers/pwm/pwm-lp3943.c index 32350a357278..61189cea1046 100644 --- a/drivers/pwm/pwm-lp3943.c +++ b/drivers/pwm/pwm-lp3943.c @@ -20,7 +20,6 @@ #define LP3943_MAX_PERIOD 1600000 struct lp3943_pwm { - struct pwm_chip chip; struct lp3943 *lp3943; struct lp3943_platform_data *pdata; struct lp3943_pwm_map pwm_map[LP3943_NUM_PWMS]; @@ -28,7 +27,7 @@ struct lp3943_pwm { static inline struct lp3943_pwm *to_lp3943_pwm(struct pwm_chip *chip) { - return container_of(chip, struct lp3943_pwm, chip); + return pwmchip_get_drvdata(chip); } static struct lp3943_pwm_map * @@ -273,12 +272,14 @@ static int lp3943_pwm_parse_dt(struct device *dev, static int lp3943_pwm_probe(struct platform_device *pdev) { struct lp3943 *lp3943 = dev_get_drvdata(pdev->dev.parent); + struct pwm_chip *chip; struct lp3943_pwm *lp3943_pwm; int ret; - lp3943_pwm = devm_kzalloc(&pdev->dev, sizeof(*lp3943_pwm), GFP_KERNEL); - if (!lp3943_pwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, LP3943_NUM_PWMS, sizeof(*lp3943_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + lp3943_pwm = to_lp3943_pwm(chip); lp3943_pwm->pdata = lp3943->pdata; if (!lp3943_pwm->pdata) { @@ -292,11 +293,9 @@ static int lp3943_pwm_probe(struct platform_device *pdev) } lp3943_pwm->lp3943 = lp3943; - lp3943_pwm->chip.dev = &pdev->dev; - lp3943_pwm->chip.ops = &lp3943_pwm_ops; - lp3943_pwm->chip.npwm = LP3943_NUM_PWMS; + chip->ops = &lp3943_pwm_ops; - return devm_pwmchip_add(&pdev->dev, &lp3943_pwm->chip); + return devm_pwmchip_add(&pdev->dev, chip); } #ifdef CONFIG_OF diff --git a/drivers/pwm/pwm-lpc18xx-sct.c b/drivers/pwm/pwm-lpc18xx-sct.c index fe891fa71a1d..04b76d257fd8 100644 --- a/drivers/pwm/pwm-lpc18xx-sct.c +++ b/drivers/pwm/pwm-lpc18xx-sct.c @@ -92,8 +92,6 @@ struct lpc18xx_pwm_data { }; struct lpc18xx_pwm_chip { - struct device *dev; - struct pwm_chip chip; void __iomem *base; struct clk *pwm_clk; unsigned long clk_rate; @@ -110,7 +108,7 @@ struct lpc18xx_pwm_chip { static inline struct lpc18xx_pwm_chip * to_lpc18xx_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct lpc18xx_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static inline void lpc18xx_pwm_writel(struct lpc18xx_pwm_chip *lpc18xx_pwm, @@ -198,7 +196,7 @@ static int lpc18xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if (period_ns < lpc18xx_pwm->min_period_ns || period_ns > lpc18xx_pwm->max_period_ns) { - dev_err(chip->dev, "period %d not in range\n", period_ns); + dev_err(pwmchip_parent(chip), "period %d not in range\n", period_ns); return -ERANGE; } @@ -214,7 +212,7 @@ static int lpc18xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, */ if (requested_events > 2 && lpc18xx_pwm->period_ns != period_ns && lpc18xx_pwm->period_ns) { - dev_err(chip->dev, "conflicting period requested for PWM %u\n", + dev_err(pwmchip_parent(chip), "conflicting period requested for PWM %u\n", pwm->hwpwm); mutex_unlock(&lpc18xx_pwm->period_lock); return -EBUSY; @@ -289,7 +287,7 @@ static int lpc18xx_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) LPC18XX_PWM_EVENT_MAX); if (event >= LPC18XX_PWM_EVENT_MAX) { - dev_err(lpc18xx_pwm->dev, + dev_err(pwmchip_parent(chip), "maximum number of simultaneous channels reached\n"); return -EBUSY; } @@ -349,16 +347,15 @@ MODULE_DEVICE_TABLE(of, lpc18xx_pwm_of_match); static int lpc18xx_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct lpc18xx_pwm_chip *lpc18xx_pwm; int ret; u64 val; - lpc18xx_pwm = devm_kzalloc(&pdev->dev, sizeof(*lpc18xx_pwm), - GFP_KERNEL); - if (!lpc18xx_pwm) - return -ENOMEM; - - lpc18xx_pwm->dev = &pdev->dev; + chip = devm_pwmchip_alloc(&pdev->dev, LPC18XX_NUM_PWMS, sizeof(*lpc18xx_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + lpc18xx_pwm = to_lpc18xx_pwm_chip(chip); lpc18xx_pwm->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(lpc18xx_pwm->base)) @@ -389,9 +386,7 @@ static int lpc18xx_pwm_probe(struct platform_device *pdev) lpc18xx_pwm->min_period_ns = DIV_ROUND_UP(NSEC_PER_SEC, lpc18xx_pwm->clk_rate); - lpc18xx_pwm->chip.dev = &pdev->dev; - lpc18xx_pwm->chip.ops = &lpc18xx_pwm_ops; - lpc18xx_pwm->chip.npwm = LPC18XX_NUM_PWMS; + chip->ops = &lpc18xx_pwm_ops; /* SCT counter must be in unify (32 bit) mode */ lpc18xx_pwm_writel(lpc18xx_pwm, LPC18XX_PWM_CONFIG, @@ -423,21 +418,22 @@ static int lpc18xx_pwm_probe(struct platform_device *pdev) val |= LPC18XX_PWM_PRE(0); lpc18xx_pwm_writel(lpc18xx_pwm, LPC18XX_PWM_CTRL, val); - ret = pwmchip_add(&lpc18xx_pwm->chip); + ret = pwmchip_add(chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "pwmchip_add failed\n"); - platform_set_drvdata(pdev, lpc18xx_pwm); + platform_set_drvdata(pdev, chip); return 0; } static void lpc18xx_pwm_remove(struct platform_device *pdev) { - struct lpc18xx_pwm_chip *lpc18xx_pwm = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct lpc18xx_pwm_chip *lpc18xx_pwm = to_lpc18xx_pwm_chip(chip); u32 val; - pwmchip_remove(&lpc18xx_pwm->chip); + pwmchip_remove(chip); val = lpc18xx_pwm_readl(lpc18xx_pwm, LPC18XX_PWM_CTRL); lpc18xx_pwm_writel(lpc18xx_pwm, LPC18XX_PWM_CTRL, diff --git a/drivers/pwm/pwm-lpc32xx.c b/drivers/pwm/pwm-lpc32xx.c index 1d9f3e7a2434..c748537e57d1 100644 --- a/drivers/pwm/pwm-lpc32xx.c +++ b/drivers/pwm/pwm-lpc32xx.c @@ -15,7 +15,6 @@ #include struct lpc32xx_pwm_chip { - struct pwm_chip chip; struct clk *clk; void __iomem *base; }; @@ -23,8 +22,10 @@ struct lpc32xx_pwm_chip { #define PWM_ENABLE BIT(31) #define PWM_PIN_LEVEL BIT(30) -#define to_lpc32xx_pwm_chip(_chip) \ - container_of(_chip, struct lpc32xx_pwm_chip, chip) +static inline struct lpc32xx_pwm_chip *to_lpc32xx_pwm_chip(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} static int lpc32xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, int duty_ns, int period_ns) @@ -119,13 +120,15 @@ static const struct pwm_ops lpc32xx_pwm_ops = { static int lpc32xx_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct lpc32xx_pwm_chip *lpc32xx; int ret; u32 val; - lpc32xx = devm_kzalloc(&pdev->dev, sizeof(*lpc32xx), GFP_KERNEL); - if (!lpc32xx) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*lpc32xx)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + lpc32xx = to_lpc32xx_pwm_chip(chip); lpc32xx->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(lpc32xx->base)) @@ -135,16 +138,14 @@ static int lpc32xx_pwm_probe(struct platform_device *pdev) if (IS_ERR(lpc32xx->clk)) return PTR_ERR(lpc32xx->clk); - lpc32xx->chip.dev = &pdev->dev; - lpc32xx->chip.ops = &lpc32xx_pwm_ops; - lpc32xx->chip.npwm = 1; + chip->ops = &lpc32xx_pwm_ops; /* If PWM is disabled, configure the output to the default value */ val = readl(lpc32xx->base); val &= ~PWM_PIN_LEVEL; writel(val, lpc32xx->base); - ret = devm_pwmchip_add(&pdev->dev, &lpc32xx->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) { dev_err(&pdev->dev, "failed to add PWM chip, error %d\n", ret); return ret; diff --git a/drivers/pwm/pwm-lpss-pci.c b/drivers/pwm/pwm-lpss-pci.c index b4134bee2863..25045c229520 100644 --- a/drivers/pwm/pwm-lpss-pci.c +++ b/drivers/pwm/pwm-lpss-pci.c @@ -18,7 +18,7 @@ static int pwm_lpss_probe_pci(struct pci_dev *pdev, const struct pci_device_id *id) { const struct pwm_lpss_boardinfo *info; - struct pwm_lpss_chip *lpwm; + struct pwm_chip *chip; int err; err = pcim_enable_device(pdev); @@ -30,11 +30,9 @@ static int pwm_lpss_probe_pci(struct pci_dev *pdev, return err; info = (struct pwm_lpss_boardinfo *)id->driver_data; - lpwm = devm_pwm_lpss_probe(&pdev->dev, pcim_iomap_table(pdev)[0], info); - if (IS_ERR(lpwm)) - return PTR_ERR(lpwm); - - pci_set_drvdata(pdev, lpwm); + chip = devm_pwm_lpss_probe(&pdev->dev, pcim_iomap_table(pdev)[0], info); + if (IS_ERR(chip)) + return PTR_ERR(chip); pm_runtime_put(&pdev->dev); pm_runtime_allow(&pdev->dev); diff --git a/drivers/pwm/pwm-lpss-platform.c b/drivers/pwm/pwm-lpss-platform.c index 319809aac2c4..dbc9f5b17bdc 100644 --- a/drivers/pwm/pwm-lpss-platform.c +++ b/drivers/pwm/pwm-lpss-platform.c @@ -20,7 +20,7 @@ static int pwm_lpss_probe_platform(struct platform_device *pdev) { const struct pwm_lpss_boardinfo *info; - struct pwm_lpss_chip *lpwm; + struct pwm_chip *chip; void __iomem *base; info = device_get_match_data(&pdev->dev); @@ -31,11 +31,9 @@ static int pwm_lpss_probe_platform(struct platform_device *pdev) if (IS_ERR(base)) return PTR_ERR(base); - lpwm = devm_pwm_lpss_probe(&pdev->dev, base, info); - if (IS_ERR(lpwm)) - return PTR_ERR(lpwm); - - platform_set_drvdata(pdev, lpwm); + chip = devm_pwm_lpss_probe(&pdev->dev, base, info); + if (IS_ERR(chip)) + return PTR_ERR(chip); /* * On Cherry Trail devices the GFX0._PS0 AML checks if the controller diff --git a/drivers/pwm/pwm-lpss.c b/drivers/pwm/pwm-lpss.c index a6ea3ce7e019..867e2bc8c601 100644 --- a/drivers/pwm/pwm-lpss.c +++ b/drivers/pwm/pwm-lpss.c @@ -68,7 +68,7 @@ EXPORT_SYMBOL_GPL(pwm_lpss_tng_info); static inline struct pwm_lpss_chip *to_lpwm(struct pwm_chip *chip) { - return container_of(chip, struct pwm_lpss_chip, chip); + return pwmchip_get_drvdata(chip); } static inline u32 pwm_lpss_read(const struct pwm_device *pwm) @@ -106,7 +106,7 @@ static int pwm_lpss_wait_for_update(struct pwm_device *pwm) */ err = readl_poll_timeout(addr, val, !(val & PWM_SW_UPDATE), 40, ms); if (err) - dev_err(pwm->chip->dev, "PWM_SW_UPDATE was not cleared\n"); + dev_err(pwmchip_parent(pwm->chip), "PWM_SW_UPDATE was not cleared\n"); return err; } @@ -114,7 +114,7 @@ static int pwm_lpss_wait_for_update(struct pwm_device *pwm) static inline int pwm_lpss_is_updating(struct pwm_device *pwm) { if (pwm_lpss_read(pwm) & PWM_SW_UPDATE) { - dev_err(pwm->chip->dev, "PWM_SW_UPDATE is still set, skipping update\n"); + dev_err(pwmchip_parent(pwm->chip), "PWM_SW_UPDATE is still set, skipping update\n"); return -EBUSY; } @@ -190,16 +190,16 @@ static int pwm_lpss_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (state->enabled) { if (!pwm_is_enabled(pwm)) { - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); ret = pwm_lpss_prepare_enable(lpwm, pwm, state); if (ret) - pm_runtime_put(chip->dev); + pm_runtime_put(pwmchip_parent(chip)); } else { ret = pwm_lpss_prepare_enable(lpwm, pwm, state); } } else if (pwm_is_enabled(pwm)) { pwm_lpss_write(pwm, pwm_lpss_read(pwm) & ~PWM_ENABLE); - pm_runtime_put(chip->dev); + pm_runtime_put(pwmchip_parent(chip)); } return ret; @@ -213,7 +213,7 @@ static int pwm_lpss_get_state(struct pwm_chip *chip, struct pwm_device *pwm, unsigned long long base_unit, freq, on_time_div; u32 ctrl; - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); base_unit_range = BIT(lpwm->info->base_unit_bits); @@ -235,7 +235,7 @@ static int pwm_lpss_get_state(struct pwm_chip *chip, struct pwm_device *pwm, state->polarity = PWM_POLARITY_NORMAL; state->enabled = !!(ctrl & PWM_ENABLE); - pm_runtime_put(chip->dev); + pm_runtime_put(pwmchip_parent(chip)); return 0; } @@ -245,10 +245,11 @@ static const struct pwm_ops pwm_lpss_ops = { .get_state = pwm_lpss_get_state, }; -struct pwm_lpss_chip *devm_pwm_lpss_probe(struct device *dev, void __iomem *base, - const struct pwm_lpss_boardinfo *info) +struct pwm_chip *devm_pwm_lpss_probe(struct device *dev, void __iomem *base, + const struct pwm_lpss_boardinfo *info) { struct pwm_lpss_chip *lpwm; + struct pwm_chip *chip; unsigned long c; int i, ret; u32 ctrl; @@ -256,9 +257,10 @@ struct pwm_lpss_chip *devm_pwm_lpss_probe(struct device *dev, void __iomem *base if (WARN_ON(info->npwm > LPSS_MAX_PWMS)) return ERR_PTR(-ENODEV); - lpwm = devm_kzalloc(dev, sizeof(*lpwm), GFP_KERNEL); - if (!lpwm) - return ERR_PTR(-ENOMEM); + chip = devm_pwmchip_alloc(dev, info->npwm, sizeof(*lpwm)); + if (IS_ERR(chip)) + return chip; + lpwm = to_lpwm(chip); lpwm->regs = base; lpwm->info = info; @@ -267,23 +269,21 @@ struct pwm_lpss_chip *devm_pwm_lpss_probe(struct device *dev, void __iomem *base if (!c) return ERR_PTR(-EINVAL); - lpwm->chip.dev = dev; - lpwm->chip.ops = &pwm_lpss_ops; - lpwm->chip.npwm = info->npwm; + chip->ops = &pwm_lpss_ops; - ret = devm_pwmchip_add(dev, &lpwm->chip); + ret = devm_pwmchip_add(dev, chip); if (ret) { dev_err(dev, "failed to add PWM chip: %d\n", ret); return ERR_PTR(ret); } for (i = 0; i < lpwm->info->npwm; i++) { - ctrl = pwm_lpss_read(&lpwm->chip.pwms[i]); + ctrl = pwm_lpss_read(&chip->pwms[i]); if (ctrl & PWM_ENABLE) pm_runtime_get(dev); } - return lpwm; + return chip; } EXPORT_SYMBOL_GPL(devm_pwm_lpss_probe); diff --git a/drivers/pwm/pwm-lpss.h b/drivers/pwm/pwm-lpss.h index bf841250385f..b5267ab5193b 100644 --- a/drivers/pwm/pwm-lpss.h +++ b/drivers/pwm/pwm-lpss.h @@ -18,7 +18,6 @@ #define LPSS_MAX_PWMS 4 struct pwm_lpss_chip { - struct pwm_chip chip; void __iomem *regs; const struct pwm_lpss_boardinfo *info; }; diff --git a/drivers/pwm/pwm-mediatek.c b/drivers/pwm/pwm-mediatek.c index 17d290f847af..19a87873ad60 100644 --- a/drivers/pwm/pwm-mediatek.c +++ b/drivers/pwm/pwm-mediatek.c @@ -42,16 +42,13 @@ struct pwm_mediatek_of_data { /** * struct pwm_mediatek_chip - struct representing PWM chip - * @chip: linux PWM chip representation * @regs: base address of PWM chip * @clk_top: the top clock generator * @clk_main: the clock used by PWM core * @clk_pwms: the clock used by each PWM channel - * @clk_freq: the fix clock frequency of legacy MIPS SoC * @soc: pointer to chip's platform data */ struct pwm_mediatek_chip { - struct pwm_chip chip; void __iomem *regs; struct clk *clk_top; struct clk *clk_main; @@ -70,7 +67,7 @@ static const unsigned int mtk_pwm_reg_offset_v2[] = { static inline struct pwm_mediatek_chip * to_pwm_mediatek_chip(struct pwm_chip *chip) { - return container_of(chip, struct pwm_mediatek_chip, chip); + return pwmchip_get_drvdata(chip); } static int pwm_mediatek_clk_enable(struct pwm_chip *chip, @@ -150,7 +147,7 @@ static int pwm_mediatek_config(struct pwm_chip *chip, struct pwm_device *pwm, if (clkdiv > PWM_CLK_DIV_MAX) { pwm_mediatek_clk_disable(chip, pwm); - dev_err(chip->dev, "period of %d ns not supported\n", period_ns); + dev_err(pwmchip_parent(chip), "period of %d ns not supported\n", period_ns); return -EINVAL; } @@ -233,21 +230,26 @@ static const struct pwm_ops pwm_mediatek_ops = { static int pwm_mediatek_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct pwm_mediatek_chip *pc; + const struct pwm_mediatek_of_data *soc; unsigned int i; int ret; - pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL); - if (!pc) - return -ENOMEM; + soc = of_device_get_match_data(&pdev->dev); - pc->soc = of_device_get_match_data(&pdev->dev); + chip = devm_pwmchip_alloc(&pdev->dev, soc->num_pwms, sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_pwm_mediatek_chip(chip); + + pc->soc = soc; pc->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->regs)) return PTR_ERR(pc->regs); - pc->clk_pwms = devm_kmalloc_array(&pdev->dev, pc->soc->num_pwms, + pc->clk_pwms = devm_kmalloc_array(&pdev->dev, soc->num_pwms, sizeof(*pc->clk_pwms), GFP_KERNEL); if (!pc->clk_pwms) return -ENOMEM; @@ -262,7 +264,7 @@ static int pwm_mediatek_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(pc->clk_main), "Failed to get main clock\n"); - for (i = 0; i < pc->soc->num_pwms; i++) { + for (i = 0; i < soc->num_pwms; i++) { char name[8]; snprintf(name, sizeof(name), "pwm%d", i + 1); @@ -273,11 +275,9 @@ static int pwm_mediatek_probe(struct platform_device *pdev) "Failed to get %s clock\n", name); } - pc->chip.dev = &pdev->dev; - pc->chip.ops = &pwm_mediatek_ops; - pc->chip.npwm = pc->soc->num_pwms; + chip->ops = &pwm_mediatek_ops; - ret = devm_pwmchip_add(&pdev->dev, &pc->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "pwmchip_add() failed\n"); @@ -340,6 +340,13 @@ static const struct pwm_mediatek_of_data mt7986_pwm_data = { .reg_offset = mtk_pwm_reg_offset_v1, }; +static const struct pwm_mediatek_of_data mt7988_pwm_data = { + .num_pwms = 8, + .pwm45_fixup = false, + .has_ck_26m_sel = false, + .reg_offset = mtk_pwm_reg_offset_v2, +}; + static const struct pwm_mediatek_of_data mt8183_pwm_data = { .num_pwms = 4, .pwm45_fixup = false, @@ -370,6 +377,7 @@ static const struct of_device_id pwm_mediatek_of_match[] = { { .compatible = "mediatek,mt7629-pwm", .data = &mt7629_pwm_data }, { .compatible = "mediatek,mt7981-pwm", .data = &mt7981_pwm_data }, { .compatible = "mediatek,mt7986-pwm", .data = &mt7986_pwm_data }, + { .compatible = "mediatek,mt7988-pwm", .data = &mt7988_pwm_data }, { .compatible = "mediatek,mt8183-pwm", .data = &mt8183_pwm_data }, { .compatible = "mediatek,mt8365-pwm", .data = &mt8365_pwm_data }, { .compatible = "mediatek,mt8516-pwm", .data = &mt8516_pwm_data }, diff --git a/drivers/pwm/pwm-meson.c b/drivers/pwm/pwm-meson.c index 2971bbf3b5e7..a02fdbc61256 100644 --- a/drivers/pwm/pwm-meson.c +++ b/drivers/pwm/pwm-meson.c @@ -60,7 +60,7 @@ #define MISC_A_EN BIT(0) #define MESON_NUM_PWMS 2 -#define MESON_MAX_MUX_PARENTS 4 +#define MESON_NUM_MUX_PARENTS 4 static struct meson_pwm_channel_data { u8 reg_offset; @@ -97,12 +97,10 @@ struct meson_pwm_channel { }; struct meson_pwm_data { - const char * const *parent_names; - unsigned int num_parents; + const char *const parent_names[MESON_NUM_MUX_PARENTS]; }; struct meson_pwm { - struct pwm_chip chip; const struct meson_pwm_data *data; struct meson_pwm_channel channels[MESON_NUM_PWMS]; void __iomem *base; @@ -115,14 +113,14 @@ struct meson_pwm { static inline struct meson_pwm *to_meson_pwm(struct pwm_chip *chip) { - return container_of(chip, struct meson_pwm, chip); + return pwmchip_get_drvdata(chip); } static int meson_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) { struct meson_pwm *meson = to_meson_pwm(chip); struct meson_pwm_channel *channel = &meson->channels[pwm->hwpwm]; - struct device *dev = chip->dev; + struct device *dev = pwmchip_parent(chip); int err; err = clk_prepare_enable(channel->clk); @@ -143,9 +141,10 @@ static void meson_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) clk_disable_unprepare(channel->clk); } -static int meson_pwm_calc(struct meson_pwm *meson, struct pwm_device *pwm, +static int meson_pwm_calc(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { + struct meson_pwm *meson = to_meson_pwm(chip); struct meson_pwm_channel *channel = &meson->channels[pwm->hwpwm]; unsigned int cnt, duty_cnt; unsigned long fin_freq; @@ -169,19 +168,19 @@ static int meson_pwm_calc(struct meson_pwm *meson, struct pwm_device *pwm, fin_freq = clk_round_rate(channel->clk, freq); if (fin_freq == 0) { - dev_err(meson->chip.dev, "invalid source clock frequency\n"); + dev_err(pwmchip_parent(chip), "invalid source clock frequency\n"); return -EINVAL; } - dev_dbg(meson->chip.dev, "fin_freq: %lu Hz\n", fin_freq); + dev_dbg(pwmchip_parent(chip), "fin_freq: %lu Hz\n", fin_freq); cnt = div_u64(fin_freq * period, NSEC_PER_SEC); if (cnt > 0xffff) { - dev_err(meson->chip.dev, "unable to get period cnt\n"); + dev_err(pwmchip_parent(chip), "unable to get period cnt\n"); return -EINVAL; } - dev_dbg(meson->chip.dev, "period=%llu cnt=%u\n", period, cnt); + dev_dbg(pwmchip_parent(chip), "period=%llu cnt=%u\n", period, cnt); if (duty == period) { channel->hi = cnt; @@ -192,7 +191,7 @@ static int meson_pwm_calc(struct meson_pwm *meson, struct pwm_device *pwm, } else { duty_cnt = div_u64(fin_freq * duty, NSEC_PER_SEC); - dev_dbg(meson->chip.dev, "duty=%llu duty_cnt=%u\n", duty, duty_cnt); + dev_dbg(pwmchip_parent(chip), "duty=%llu duty_cnt=%u\n", duty, duty_cnt); channel->hi = duty_cnt; channel->lo = cnt - duty_cnt; @@ -203,8 +202,9 @@ static int meson_pwm_calc(struct meson_pwm *meson, struct pwm_device *pwm, return 0; } -static void meson_pwm_enable(struct meson_pwm *meson, struct pwm_device *pwm) +static void meson_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) { + struct meson_pwm *meson = to_meson_pwm(chip); struct meson_pwm_channel *channel = &meson->channels[pwm->hwpwm]; struct meson_pwm_channel_data *channel_data; unsigned long flags; @@ -215,7 +215,7 @@ static void meson_pwm_enable(struct meson_pwm *meson, struct pwm_device *pwm) err = clk_set_rate(channel->clk, channel->rate); if (err) - dev_err(meson->chip.dev, "setting clock rate failed\n"); + dev_err(pwmchip_parent(chip), "setting clock rate failed\n"); spin_lock_irqsave(&meson->lock, flags); @@ -230,8 +230,9 @@ static void meson_pwm_enable(struct meson_pwm *meson, struct pwm_device *pwm) spin_unlock_irqrestore(&meson->lock, flags); } -static void meson_pwm_disable(struct meson_pwm *meson, struct pwm_device *pwm) +static void meson_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) { + struct meson_pwm *meson = to_meson_pwm(chip); unsigned long flags; u32 value; @@ -269,16 +270,16 @@ static int meson_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, channel->hi = ~0; channel->lo = 0; - meson_pwm_enable(meson, pwm); + meson_pwm_enable(chip, pwm); } else { - meson_pwm_disable(meson, pwm); + meson_pwm_disable(chip, pwm); } } else { - err = meson_pwm_calc(meson, pwm, state); + err = meson_pwm_calc(chip, pwm, state); if (err < 0) return err; - meson_pwm_enable(meson, pwm); + meson_pwm_enable(chip, pwm); } return 0; @@ -337,62 +338,32 @@ static const struct pwm_ops meson_pwm_ops = { .get_state = meson_pwm_get_state, }; -static const char * const pwm_meson8b_parent_names[] = { - "xtal", NULL, "fclk_div4", "fclk_div3" -}; - static const struct meson_pwm_data pwm_meson8b_data = { - .parent_names = pwm_meson8b_parent_names, - .num_parents = ARRAY_SIZE(pwm_meson8b_parent_names), + .parent_names = { "xtal", NULL, "fclk_div4", "fclk_div3" }, }; /* * Only the 2 first inputs of the GXBB AO PWMs are valid * The last 2 are grounded */ -static const char * const pwm_gxbb_ao_parent_names[] = { - "xtal", "clk81" -}; - static const struct meson_pwm_data pwm_gxbb_ao_data = { - .parent_names = pwm_gxbb_ao_parent_names, - .num_parents = ARRAY_SIZE(pwm_gxbb_ao_parent_names), -}; - -static const char * const pwm_axg_ee_parent_names[] = { - "xtal", "fclk_div5", "fclk_div4", "fclk_div3" + .parent_names = { "xtal", "clk81", NULL, NULL }, }; static const struct meson_pwm_data pwm_axg_ee_data = { - .parent_names = pwm_axg_ee_parent_names, - .num_parents = ARRAY_SIZE(pwm_axg_ee_parent_names), -}; - -static const char * const pwm_axg_ao_parent_names[] = { - "xtal", "axg_ao_clk81", "fclk_div4", "fclk_div5" + .parent_names = { "xtal", "fclk_div5", "fclk_div4", "fclk_div3" }, }; static const struct meson_pwm_data pwm_axg_ao_data = { - .parent_names = pwm_axg_ao_parent_names, - .num_parents = ARRAY_SIZE(pwm_axg_ao_parent_names), -}; - -static const char * const pwm_g12a_ao_ab_parent_names[] = { - "xtal", "g12a_ao_clk81", "fclk_div4", "fclk_div5" + .parent_names = { "xtal", "axg_ao_clk81", "fclk_div4", "fclk_div5" }, }; static const struct meson_pwm_data pwm_g12a_ao_ab_data = { - .parent_names = pwm_g12a_ao_ab_parent_names, - .num_parents = ARRAY_SIZE(pwm_g12a_ao_ab_parent_names), -}; - -static const char * const pwm_g12a_ao_cd_parent_names[] = { - "xtal", "g12a_ao_clk81", + .parent_names = { "xtal", "g12a_ao_clk81", "fclk_div4", "fclk_div5" }, }; static const struct meson_pwm_data pwm_g12a_ao_cd_data = { - .parent_names = pwm_g12a_ao_cd_parent_names, - .num_parents = ARRAY_SIZE(pwm_g12a_ao_cd_parent_names), + .parent_names = { "xtal", "g12a_ao_clk81", NULL, NULL }, }; static const struct of_device_id meson_pwm_matches[] = { @@ -432,20 +403,21 @@ static const struct of_device_id meson_pwm_matches[] = { }; MODULE_DEVICE_TABLE(of, meson_pwm_matches); -static int meson_pwm_init_channels(struct meson_pwm *meson) +static int meson_pwm_init_channels(struct pwm_chip *chip) { - struct clk_parent_data mux_parent_data[MESON_MAX_MUX_PARENTS] = {}; - struct device *dev = meson->chip.dev; + struct meson_pwm *meson = to_meson_pwm(chip); + struct clk_parent_data mux_parent_data[MESON_NUM_MUX_PARENTS] = {}; + struct device *dev = pwmchip_parent(chip); unsigned int i; char name[255]; int err; - for (i = 0; i < meson->data->num_parents; i++) { + for (i = 0; i < MESON_NUM_MUX_PARENTS; i++) { mux_parent_data[i].index = -1; mux_parent_data[i].name = meson->data->parent_names[i]; } - for (i = 0; i < meson->chip.npwm; i++) { + for (i = 0; i < chip->npwm; i++) { struct meson_pwm_channel *channel = &meson->channels[i]; struct clk_parent_data div_parent = {}, gate_parent = {}; struct clk_init_data init = {}; @@ -456,7 +428,7 @@ static int meson_pwm_init_channels(struct meson_pwm *meson) init.ops = &clk_mux_ops; init.flags = 0; init.parent_data = mux_parent_data; - init.num_parents = meson->data->num_parents; + init.num_parents = MESON_NUM_MUX_PARENTS; channel->mux.reg = meson->base + REG_MISC_AB; channel->mux.shift = @@ -525,29 +497,29 @@ static int meson_pwm_init_channels(struct meson_pwm *meson) static int meson_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct meson_pwm *meson; int err; - meson = devm_kzalloc(&pdev->dev, sizeof(*meson), GFP_KERNEL); - if (!meson) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, MESON_NUM_PWMS, sizeof(*meson)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + meson = to_meson_pwm(chip); meson->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(meson->base)) return PTR_ERR(meson->base); spin_lock_init(&meson->lock); - meson->chip.dev = &pdev->dev; - meson->chip.ops = &meson_pwm_ops; - meson->chip.npwm = MESON_NUM_PWMS; + chip->ops = &meson_pwm_ops; meson->data = of_device_get_match_data(&pdev->dev); - err = meson_pwm_init_channels(meson); + err = meson_pwm_init_channels(chip); if (err < 0) return err; - err = devm_pwmchip_add(&pdev->dev, &meson->chip); + err = devm_pwmchip_add(&pdev->dev, chip); if (err < 0) return dev_err_probe(&pdev->dev, err, "failed to register PWM chip\n"); diff --git a/drivers/pwm/pwm-microchip-core.c b/drivers/pwm/pwm-microchip-core.c index c0c53968f3e9..c1f2287b8e97 100644 --- a/drivers/pwm/pwm-microchip-core.c +++ b/drivers/pwm/pwm-microchip-core.c @@ -54,7 +54,6 @@ #define MCHPCOREPWM_TIMEOUT_MS 100u struct mchp_core_pwm_chip { - struct pwm_chip chip; struct clk *clk; void __iomem *base; struct mutex lock; /* protects the shared period */ @@ -65,7 +64,7 @@ struct mchp_core_pwm_chip { static inline struct mchp_core_pwm_chip *to_mchp_core_pwm(struct pwm_chip *chip) { - return container_of(chip, struct mchp_core_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static void mchp_core_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm, @@ -447,13 +446,15 @@ MODULE_DEVICE_TABLE(of, mchp_core_of_match); static int mchp_core_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct mchp_core_pwm_chip *mchp_core_pwm; struct resource *regs; int ret; - mchp_core_pwm = devm_kzalloc(&pdev->dev, sizeof(*mchp_core_pwm), GFP_KERNEL); - if (!mchp_core_pwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 16, sizeof(*mchp_core_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + mchp_core_pwm = to_mchp_core_pwm(chip); mchp_core_pwm->base = devm_platform_get_and_ioremap_resource(pdev, 0, ®s); if (IS_ERR(mchp_core_pwm->base)) @@ -470,9 +471,7 @@ static int mchp_core_pwm_probe(struct platform_device *pdev) mutex_init(&mchp_core_pwm->lock); - mchp_core_pwm->chip.dev = &pdev->dev; - mchp_core_pwm->chip.ops = &mchp_core_pwm_ops; - mchp_core_pwm->chip.npwm = 16; + chip->ops = &mchp_core_pwm_ops; mchp_core_pwm->channel_enabled = readb_relaxed(mchp_core_pwm->base + MCHPCOREPWM_EN(0)); mchp_core_pwm->channel_enabled |= @@ -485,7 +484,7 @@ static int mchp_core_pwm_probe(struct platform_device *pdev) writel_relaxed(1U, mchp_core_pwm->base + MCHPCOREPWM_SYNC_UPD); mchp_core_pwm->update_timestamp = ktime_get(); - ret = devm_pwmchip_add(&pdev->dev, &mchp_core_pwm->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret) return dev_err_probe(&pdev->dev, ret, "Failed to add pwmchip\n"); diff --git a/drivers/pwm/pwm-mtk-disp.c b/drivers/pwm/pwm-mtk-disp.c index a72f7be36996..bafd6b6195f6 100644 --- a/drivers/pwm/pwm-mtk-disp.c +++ b/drivers/pwm/pwm-mtk-disp.c @@ -42,7 +42,6 @@ struct mtk_pwm_data { }; struct mtk_disp_pwm { - struct pwm_chip chip; const struct mtk_pwm_data *data; struct clk *clk_main; struct clk *clk_mm; @@ -52,7 +51,7 @@ struct mtk_disp_pwm { static inline struct mtk_disp_pwm *to_mtk_disp_pwm(struct pwm_chip *chip) { - return container_of(chip, struct mtk_disp_pwm, chip); + return pwmchip_get_drvdata(chip); } static void mtk_disp_pwm_update_bits(struct mtk_disp_pwm *mdp, u32 offset, @@ -91,14 +90,14 @@ static int mtk_disp_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (!mdp->enabled) { err = clk_prepare_enable(mdp->clk_main); if (err < 0) { - dev_err(chip->dev, "Can't enable mdp->clk_main: %pe\n", + dev_err(pwmchip_parent(chip), "Can't enable mdp->clk_main: %pe\n", ERR_PTR(err)); return err; } err = clk_prepare_enable(mdp->clk_mm); if (err < 0) { - dev_err(chip->dev, "Can't enable mdp->clk_mm: %pe\n", + dev_err(pwmchip_parent(chip), "Can't enable mdp->clk_mm: %pe\n", ERR_PTR(err)); clk_disable_unprepare(mdp->clk_main); return err; @@ -181,13 +180,13 @@ static int mtk_disp_pwm_get_state(struct pwm_chip *chip, err = clk_prepare_enable(mdp->clk_main); if (err < 0) { - dev_err(chip->dev, "Can't enable mdp->clk_main: %pe\n", ERR_PTR(err)); + dev_err(pwmchip_parent(chip), "Can't enable mdp->clk_main: %pe\n", ERR_PTR(err)); return err; } err = clk_prepare_enable(mdp->clk_mm); if (err < 0) { - dev_err(chip->dev, "Can't enable mdp->clk_mm: %pe\n", ERR_PTR(err)); + dev_err(pwmchip_parent(chip), "Can't enable mdp->clk_mm: %pe\n", ERR_PTR(err)); clk_disable_unprepare(mdp->clk_main); return err; } @@ -231,12 +230,14 @@ static const struct pwm_ops mtk_disp_pwm_ops = { static int mtk_disp_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct mtk_disp_pwm *mdp; int ret; - mdp = devm_kzalloc(&pdev->dev, sizeof(*mdp), GFP_KERNEL); - if (!mdp) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*mdp)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + mdp = to_mtk_disp_pwm(chip); mdp->data = of_device_get_match_data(&pdev->dev); @@ -254,11 +255,9 @@ static int mtk_disp_pwm_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(mdp->clk_mm), "Failed to get mm clock\n"); - mdp->chip.dev = &pdev->dev; - mdp->chip.ops = &mtk_disp_pwm_ops; - mdp->chip.npwm = 1; + chip->ops = &mtk_disp_pwm_ops; - ret = devm_pwmchip_add(&pdev->dev, &mdp->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "pwmchip_add() failed\n"); diff --git a/drivers/pwm/pwm-mxs.c b/drivers/pwm/pwm-mxs.c index 1b5e787d78f1..8cad214b1c29 100644 --- a/drivers/pwm/pwm-mxs.c +++ b/drivers/pwm/pwm-mxs.c @@ -37,12 +37,14 @@ static const u8 cdiv_shift[PERIOD_CDIV_MAX] = { }; struct mxs_pwm_chip { - struct pwm_chip chip; struct clk *clk; void __iomem *base; }; -#define to_mxs_pwm_chip(_chip) container_of(_chip, struct mxs_pwm_chip, chip) +static inline struct mxs_pwm_chip *to_mxs_pwm_chip(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} static int mxs_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) @@ -120,12 +122,21 @@ static const struct pwm_ops mxs_pwm_ops = { static int mxs_pwm_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; + struct pwm_chip *chip; struct mxs_pwm_chip *mxs; + u32 npwm; int ret; - mxs = devm_kzalloc(&pdev->dev, sizeof(*mxs), GFP_KERNEL); - if (!mxs) - return -ENOMEM; + ret = of_property_read_u32(np, "fsl,pwm-number", &npwm); + if (ret < 0) { + dev_err(&pdev->dev, "failed to get pwm number: %d\n", ret); + return ret; + } + + chip = devm_pwmchip_alloc(&pdev->dev, npwm, sizeof(*mxs)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + mxs = to_mxs_pwm_chip(chip); mxs->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(mxs->base)) @@ -135,21 +146,14 @@ static int mxs_pwm_probe(struct platform_device *pdev) if (IS_ERR(mxs->clk)) return PTR_ERR(mxs->clk); - mxs->chip.dev = &pdev->dev; - mxs->chip.ops = &mxs_pwm_ops; - - ret = of_property_read_u32(np, "fsl,pwm-number", &mxs->chip.npwm); - if (ret < 0) { - dev_err(&pdev->dev, "failed to get pwm number: %d\n", ret); - return ret; - } + chip->ops = &mxs_pwm_ops; /* FIXME: Only do this if the PWM isn't already running */ ret = stmp_reset_block(mxs->base); if (ret) return dev_err_probe(&pdev->dev, ret, "failed to reset PWM\n"); - ret = devm_pwmchip_add(&pdev->dev, &mxs->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) { dev_err(&pdev->dev, "failed to add pwm chip %d\n", ret); return ret; diff --git a/drivers/pwm/pwm-ntxec.c b/drivers/pwm/pwm-ntxec.c index 78606039eda2..28d1c2e5a98f 100644 --- a/drivers/pwm/pwm-ntxec.c +++ b/drivers/pwm/pwm-ntxec.c @@ -25,12 +25,11 @@ struct ntxec_pwm { struct ntxec *ec; - struct pwm_chip chip; }; static struct ntxec_pwm *ntxec_pwm_from_chip(struct pwm_chip *chip) { - return container_of(chip, struct ntxec_pwm, chip); + return pwmchip_get_drvdata(chip); } #define NTXEC_REG_AUTO_OFF_HI 0xa1 @@ -141,16 +140,13 @@ static int ntxec_pwm_probe(struct platform_device *pdev) device_set_of_node_from_dev(&pdev->dev, pdev->dev.parent); - priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*priv)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + priv = ntxec_pwm_from_chip(chip); priv->ec = ec; - - chip = &priv->chip; - chip->dev = &pdev->dev; chip->ops = &ntxec_pwm_ops; - chip->npwm = 1; return devm_pwmchip_add(&pdev->dev, chip); } diff --git a/drivers/pwm/pwm-omap-dmtimer.c b/drivers/pwm/pwm-omap-dmtimer.c index 496bd73d29fe..cd51c4a938f5 100644 --- a/drivers/pwm/pwm-omap-dmtimer.c +++ b/drivers/pwm/pwm-omap-dmtimer.c @@ -53,13 +53,11 @@ /** * struct pwm_omap_dmtimer_chip - Structure representing a pwm chip * corresponding to omap dmtimer. - * @chip: PWM chip structure representing PWM controller * @dm_timer: Pointer to omap dm timer. * @pdata: Pointer to omap dm timer ops. * @dm_timer_pdev: Pointer to omap dm timer platform device */ struct pwm_omap_dmtimer_chip { - struct pwm_chip chip; /* Mutex to protect pwm apply state */ struct omap_dm_timer *dm_timer; const struct omap_dm_timer_ops *pdata; @@ -69,7 +67,7 @@ struct pwm_omap_dmtimer_chip { static inline struct pwm_omap_dmtimer_chip * to_pwm_omap_dmtimer_chip(struct pwm_chip *chip) { - return container_of(chip, struct pwm_omap_dmtimer_chip, chip); + return pwmchip_get_drvdata(chip); } /** @@ -155,7 +153,7 @@ static int pwm_omap_dmtimer_config(struct pwm_chip *chip, unsigned long clk_rate; struct clk *fclk; - dev_dbg(chip->dev, "requested duty cycle: %d ns, period: %d ns\n", + dev_dbg(pwmchip_parent(chip), "requested duty cycle: %d ns, period: %d ns\n", duty_ns, period_ns); if (duty_ns == pwm_get_duty_cycle(pwm) && @@ -164,17 +162,17 @@ static int pwm_omap_dmtimer_config(struct pwm_chip *chip, fclk = omap->pdata->get_fclk(omap->dm_timer); if (!fclk) { - dev_err(chip->dev, "invalid pmtimer fclk\n"); + dev_err(pwmchip_parent(chip), "invalid pmtimer fclk\n"); return -EINVAL; } clk_rate = clk_get_rate(fclk); if (!clk_rate) { - dev_err(chip->dev, "invalid pmtimer fclk rate\n"); + dev_err(pwmchip_parent(chip), "invalid pmtimer fclk rate\n"); return -EINVAL; } - dev_dbg(chip->dev, "clk rate: %luHz\n", clk_rate); + dev_dbg(pwmchip_parent(chip), "clk rate: %luHz\n", clk_rate); /* * Calculate the appropriate load and match values based on the @@ -196,27 +194,27 @@ static int pwm_omap_dmtimer_config(struct pwm_chip *chip, duty_cycles = pwm_omap_dmtimer_get_clock_cycles(clk_rate, duty_ns); if (period_cycles < 2) { - dev_info(chip->dev, + dev_info(pwmchip_parent(chip), "period %d ns too short for clock rate %lu Hz\n", period_ns, clk_rate); return -EINVAL; } if (duty_cycles < 1) { - dev_dbg(chip->dev, + dev_dbg(pwmchip_parent(chip), "duty cycle %d ns is too short for clock rate %lu Hz\n", duty_ns, clk_rate); - dev_dbg(chip->dev, "using minimum of 1 clock cycle\n"); + dev_dbg(pwmchip_parent(chip), "using minimum of 1 clock cycle\n"); duty_cycles = 1; } else if (duty_cycles >= period_cycles) { - dev_dbg(chip->dev, + dev_dbg(pwmchip_parent(chip), "duty cycle %d ns is too long for period %d ns at clock rate %lu Hz\n", duty_ns, period_ns, clk_rate); - dev_dbg(chip->dev, "using maximum of 1 clock cycle less than period\n"); + dev_dbg(pwmchip_parent(chip), "using maximum of 1 clock cycle less than period\n"); duty_cycles = period_cycles - 1; } - dev_dbg(chip->dev, "effective duty cycle: %lld ns, period: %lld ns\n", + dev_dbg(pwmchip_parent(chip), "effective duty cycle: %lld ns, period: %lld ns\n", DIV_ROUND_CLOSEST_ULL((u64)NSEC_PER_SEC * duty_cycles, clk_rate), DIV_ROUND_CLOSEST_ULL((u64)NSEC_PER_SEC * period_cycles, @@ -228,7 +226,7 @@ static int pwm_omap_dmtimer_config(struct pwm_chip *chip, omap->pdata->set_load(omap->dm_timer, load_value); omap->pdata->set_match(omap->dm_timer, true, match_value); - dev_dbg(chip->dev, "load value: %#08x (%d), match value: %#08x (%d)\n", + dev_dbg(pwmchip_parent(chip), "load value: %#08x (%d), match value: %#08x (%d)\n", load_value, load_value, match_value, match_value); return 0; @@ -311,6 +309,7 @@ static int pwm_omap_dmtimer_probe(struct platform_device *pdev) struct dmtimer_platform_data *timer_pdata; const struct omap_dm_timer_ops *pdata; struct platform_device *timer_pdev; + struct pwm_chip *chip; struct pwm_omap_dmtimer_chip *omap; struct omap_dm_timer *dm_timer; struct device_node *timer; @@ -368,11 +367,12 @@ static int pwm_omap_dmtimer_probe(struct platform_device *pdev) goto err_request_timer; } - omap = devm_kzalloc(&pdev->dev, sizeof(*omap), GFP_KERNEL); - if (!omap) { - ret = -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*omap)); + if (IS_ERR(chip)) { + ret = PTR_ERR(chip); goto err_alloc_omap; } + omap = to_pwm_omap_dmtimer_chip(chip); omap->pdata = pdata; omap->dm_timer = dm_timer; @@ -392,11 +392,9 @@ static int pwm_omap_dmtimer_probe(struct platform_device *pdev) if (!of_property_read_u32(pdev->dev.of_node, "ti,clock-source", &v)) omap->pdata->set_source(omap->dm_timer, v); - omap->chip.dev = &pdev->dev; - omap->chip.ops = &pwm_omap_dmtimer_ops; - omap->chip.npwm = 1; + chip->ops = &pwm_omap_dmtimer_ops; - ret = pwmchip_add(&omap->chip); + ret = pwmchip_add(chip); if (ret < 0) { dev_err(&pdev->dev, "failed to register PWM\n"); goto err_pwmchip_add; @@ -404,7 +402,7 @@ static int pwm_omap_dmtimer_probe(struct platform_device *pdev) of_node_put(timer); - platform_set_drvdata(pdev, omap); + platform_set_drvdata(pdev, chip); return 0; @@ -432,9 +430,10 @@ err_find_timer_pdev: static void pwm_omap_dmtimer_remove(struct platform_device *pdev) { - struct pwm_omap_dmtimer_chip *omap = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct pwm_omap_dmtimer_chip *omap = to_pwm_omap_dmtimer_chip(chip); - pwmchip_remove(&omap->chip); + pwmchip_remove(chip); if (pm_runtime_active(&omap->dm_timer_pdev->dev)) omap->pdata->stop(omap->dm_timer); diff --git a/drivers/pwm/pwm-pca9685.c b/drivers/pwm/pwm-pca9685.c index e79b1de8c4d8..c5da2a6ed846 100644 --- a/drivers/pwm/pwm-pca9685.c +++ b/drivers/pwm/pwm-pca9685.c @@ -76,7 +76,6 @@ #define REG_OFF_L(C) ((C) >= PCA9685_MAXCHAN ? PCA9685_ALL_LED_OFF_L : LED_N_OFF_L((C))) struct pca9685 { - struct pwm_chip chip; struct regmap *regmap; struct mutex lock; DECLARE_BITMAP(pwms_enabled, PCA9685_MAXCHAN + 1); @@ -88,7 +87,7 @@ struct pca9685 { static inline struct pca9685 *to_pca(struct pwm_chip *chip) { - return container_of(chip, struct pca9685, chip); + return pwmchip_get_drvdata(chip); } /* This function is supposed to be called with the lock mutex held */ @@ -107,9 +106,10 @@ static bool pca9685_prescaler_can_change(struct pca9685 *pca, int channel) return test_bit(channel, pca->pwms_enabled); } -static int pca9685_read_reg(struct pca9685 *pca, unsigned int reg, unsigned int *val) +static int pca9685_read_reg(struct pwm_chip *chip, unsigned int reg, unsigned int *val) { - struct device *dev = pca->chip.dev; + struct pca9685 *pca = to_pca(chip); + struct device *dev = pwmchip_parent(chip); int err; err = regmap_read(pca->regmap, reg, val); @@ -119,9 +119,10 @@ static int pca9685_read_reg(struct pca9685 *pca, unsigned int reg, unsigned int return err; } -static int pca9685_write_reg(struct pca9685 *pca, unsigned int reg, unsigned int val) +static int pca9685_write_reg(struct pwm_chip *chip, unsigned int reg, unsigned int val) { - struct device *dev = pca->chip.dev; + struct pca9685 *pca = to_pca(chip); + struct device *dev = pwmchip_parent(chip); int err; err = regmap_write(pca->regmap, reg, val); @@ -132,19 +133,19 @@ static int pca9685_write_reg(struct pca9685 *pca, unsigned int reg, unsigned int } /* Helper function to set the duty cycle ratio to duty/4096 (e.g. duty=2048 -> 50%) */ -static void pca9685_pwm_set_duty(struct pca9685 *pca, int channel, unsigned int duty) +static void pca9685_pwm_set_duty(struct pwm_chip *chip, int channel, unsigned int duty) { - struct pwm_device *pwm = &pca->chip.pwms[channel]; + struct pwm_device *pwm = &chip->pwms[channel]; unsigned int on, off; if (duty == 0) { /* Set the full OFF bit, which has the highest precedence */ - pca9685_write_reg(pca, REG_OFF_H(channel), LED_FULL); + pca9685_write_reg(chip, REG_OFF_H(channel), LED_FULL); return; } else if (duty >= PCA9685_COUNTER_RANGE) { /* Set the full ON bit and clear the full OFF bit */ - pca9685_write_reg(pca, REG_ON_H(channel), LED_FULL); - pca9685_write_reg(pca, REG_OFF_H(channel), 0); + pca9685_write_reg(chip, REG_ON_H(channel), LED_FULL); + pca9685_write_reg(chip, REG_OFF_H(channel), 0); return; } @@ -164,16 +165,16 @@ static void pca9685_pwm_set_duty(struct pca9685 *pca, int channel, unsigned int off = (on + duty) % PCA9685_COUNTER_RANGE; /* Set ON time (clears full ON bit) */ - pca9685_write_reg(pca, REG_ON_L(channel), on & 0xff); - pca9685_write_reg(pca, REG_ON_H(channel), (on >> 8) & 0xf); + pca9685_write_reg(chip, REG_ON_L(channel), on & 0xff); + pca9685_write_reg(chip, REG_ON_H(channel), (on >> 8) & 0xf); /* Set OFF time (clears full OFF bit) */ - pca9685_write_reg(pca, REG_OFF_L(channel), off & 0xff); - pca9685_write_reg(pca, REG_OFF_H(channel), (off >> 8) & 0xf); + pca9685_write_reg(chip, REG_OFF_L(channel), off & 0xff); + pca9685_write_reg(chip, REG_OFF_H(channel), (off >> 8) & 0xf); } -static unsigned int pca9685_pwm_get_duty(struct pca9685 *pca, int channel) +static unsigned int pca9685_pwm_get_duty(struct pwm_chip *chip, int channel) { - struct pwm_device *pwm = &pca->chip.pwms[channel]; + struct pwm_device *pwm = &chip->pwms[channel]; unsigned int off = 0, on = 0, val = 0; if (WARN_ON(channel >= PCA9685_MAXCHAN)) { @@ -181,25 +182,25 @@ static unsigned int pca9685_pwm_get_duty(struct pca9685 *pca, int channel) return 0; } - pca9685_read_reg(pca, LED_N_OFF_H(channel), &off); + pca9685_read_reg(chip, LED_N_OFF_H(channel), &off); if (off & LED_FULL) { /* Full OFF bit is set */ return 0; } - pca9685_read_reg(pca, LED_N_ON_H(channel), &on); + pca9685_read_reg(chip, LED_N_ON_H(channel), &on); if (on & LED_FULL) { /* Full ON bit is set */ return PCA9685_COUNTER_RANGE; } - pca9685_read_reg(pca, LED_N_OFF_L(channel), &val); + pca9685_read_reg(chip, LED_N_OFF_L(channel), &val); off = ((off & 0xf) << 8) | (val & 0xff); if (!pwm->state.usage_power) return off; /* Read ON register to calculate duty cycle of staggered output */ - if (pca9685_read_reg(pca, LED_N_ON_L(channel), &val)) { + if (pca9685_read_reg(chip, LED_N_ON_L(channel), &val)) { /* Reset val to 0 in case reading LED_N_ON_L failed */ val = 0; } @@ -247,35 +248,37 @@ static void pca9685_pwm_clear_inuse(struct pca9685 *pca, int pwm_idx) static int pca9685_pwm_gpio_request(struct gpio_chip *gpio, unsigned int offset) { - struct pca9685 *pca = gpiochip_get_data(gpio); + struct pwm_chip *chip = gpiochip_get_data(gpio); + struct pca9685 *pca = to_pca(chip); if (pca9685_pwm_test_and_set_inuse(pca, offset)) return -EBUSY; - pm_runtime_get_sync(pca->chip.dev); + pm_runtime_get_sync(pwmchip_parent(chip)); return 0; } static int pca9685_pwm_gpio_get(struct gpio_chip *gpio, unsigned int offset) { - struct pca9685 *pca = gpiochip_get_data(gpio); + struct pwm_chip *chip = gpiochip_get_data(gpio); - return pca9685_pwm_get_duty(pca, offset) != 0; + return pca9685_pwm_get_duty(chip, offset) != 0; } static void pca9685_pwm_gpio_set(struct gpio_chip *gpio, unsigned int offset, int value) { - struct pca9685 *pca = gpiochip_get_data(gpio); + struct pwm_chip *chip = gpiochip_get_data(gpio); - pca9685_pwm_set_duty(pca, offset, value ? PCA9685_COUNTER_RANGE : 0); + pca9685_pwm_set_duty(chip, offset, value ? PCA9685_COUNTER_RANGE : 0); } static void pca9685_pwm_gpio_free(struct gpio_chip *gpio, unsigned int offset) { - struct pca9685 *pca = gpiochip_get_data(gpio); + struct pwm_chip *chip = gpiochip_get_data(gpio); + struct pca9685 *pca = to_pca(chip); - pca9685_pwm_set_duty(pca, offset, 0); - pm_runtime_put(pca->chip.dev); + pca9685_pwm_set_duty(chip, offset, 0); + pm_runtime_put(pwmchip_parent(chip)); pca9685_pwm_clear_inuse(pca, offset); } @@ -306,9 +309,10 @@ static int pca9685_pwm_gpio_direction_output(struct gpio_chip *gpio, * expose a GPIO chip here which can exclusively take over the underlying * PWM channel. */ -static int pca9685_pwm_gpio_probe(struct pca9685 *pca) +static int pca9685_pwm_gpio_probe(struct pwm_chip *chip) { - struct device *dev = pca->chip.dev; + struct pca9685 *pca = to_pca(chip); + struct device *dev = pwmchip_parent(chip); pca->gpio.label = dev_name(dev); pca->gpio.parent = dev; @@ -323,7 +327,7 @@ static int pca9685_pwm_gpio_probe(struct pca9685 *pca) pca->gpio.ngpio = PCA9685_MAXCHAN; pca->gpio.can_sleep = true; - return devm_gpiochip_add_data(dev, &pca->gpio, pca); + return devm_gpiochip_add_data(dev, &pca->gpio, chip); } #else static inline bool pca9685_pwm_test_and_set_inuse(struct pca9685 *pca, @@ -337,15 +341,16 @@ pca9685_pwm_clear_inuse(struct pca9685 *pca, int pwm_idx) { } -static inline int pca9685_pwm_gpio_probe(struct pca9685 *pca) +static inline int pca9685_pwm_gpio_probe(struct pwm_chip *chip) { return 0; } #endif -static void pca9685_set_sleep_mode(struct pca9685 *pca, bool enable) +static void pca9685_set_sleep_mode(struct pwm_chip *chip, bool enable) { - struct device *dev = pca->chip.dev; + struct device *dev = pwmchip_parent(chip); + struct pca9685 *pca = to_pca(chip); int err = regmap_update_bits(pca->regmap, PCA9685_MODE1, MODE1_SLEEP, enable ? MODE1_SLEEP : 0); if (err) { @@ -373,19 +378,19 @@ static int __pca9685_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, prescale = DIV_ROUND_CLOSEST_ULL(PCA9685_OSC_CLOCK_MHZ * state->period, PCA9685_COUNTER_RANGE * 1000) - 1; if (prescale < PCA9685_PRESCALE_MIN || prescale > PCA9685_PRESCALE_MAX) { - dev_err(chip->dev, "pwm not changed: period out of bounds!\n"); + dev_err(pwmchip_parent(chip), "pwm not changed: period out of bounds!\n"); return -EINVAL; } if (!state->enabled) { - pca9685_pwm_set_duty(pca, pwm->hwpwm, 0); + pca9685_pwm_set_duty(chip, pwm->hwpwm, 0); return 0; } - pca9685_read_reg(pca, PCA9685_PRESCALE, &val); + pca9685_read_reg(chip, PCA9685_PRESCALE, &val); if (prescale != val) { if (!pca9685_prescaler_can_change(pca, pwm->hwpwm)) { - dev_err(chip->dev, + dev_err(pwmchip_parent(chip), "pwm not changed: periods of enabled pwms must match!\n"); return -EBUSY; } @@ -397,18 +402,18 @@ static int __pca9685_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, * state is guaranteed active here. */ /* Put chip into sleep mode */ - pca9685_set_sleep_mode(pca, true); + pca9685_set_sleep_mode(chip, true); /* Change the chip-wide output frequency */ - pca9685_write_reg(pca, PCA9685_PRESCALE, prescale); + pca9685_write_reg(chip, PCA9685_PRESCALE, prescale); /* Wake the chip up */ - pca9685_set_sleep_mode(pca, false); + pca9685_set_sleep_mode(chip, false); } duty = PCA9685_COUNTER_RANGE * state->duty_cycle; duty = DIV_ROUND_UP_ULL(duty, state->period); - pca9685_pwm_set_duty(pca, pwm->hwpwm, duty); + pca9685_pwm_set_duty(chip, pwm->hwpwm, duty); return 0; } @@ -434,12 +439,11 @@ static int pca9685_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, static int pca9685_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, struct pwm_state *state) { - struct pca9685 *pca = to_pca(chip); unsigned long long duty; unsigned int val = 0; /* Calculate (chip-wide) period from prescale value */ - pca9685_read_reg(pca, PCA9685_PRESCALE, &val); + pca9685_read_reg(chip, PCA9685_PRESCALE, &val); /* * PCA9685_OSC_CLOCK_MHZ is 25, i.e. an integer divider of 1000. * The following calculation is therefore only a multiplication @@ -462,7 +466,7 @@ static int pca9685_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, } state->enabled = true; - duty = pca9685_pwm_get_duty(pca, pwm->hwpwm); + duty = pca9685_pwm_get_duty(chip, pwm->hwpwm); state->duty_cycle = DIV_ROUND_DOWN_ULL(duty * state->period, PCA9685_COUNTER_RANGE); return 0; @@ -482,7 +486,7 @@ static int pca9685_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) mutex_unlock(&pca->lock); } - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); return 0; } @@ -492,11 +496,11 @@ static void pca9685_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) struct pca9685 *pca = to_pca(chip); mutex_lock(&pca->lock); - pca9685_pwm_set_duty(pca, pwm->hwpwm, 0); + pca9685_pwm_set_duty(chip, pwm->hwpwm, 0); clear_bit(pwm->hwpwm, pca->pwms_enabled); mutex_unlock(&pca->lock); - pm_runtime_put(chip->dev); + pm_runtime_put(pwmchip_parent(chip)); pca9685_pwm_clear_inuse(pca, pwm->hwpwm); } @@ -516,13 +520,16 @@ static const struct regmap_config pca9685_regmap_i2c_config = { static int pca9685_pwm_probe(struct i2c_client *client) { + struct pwm_chip *chip; struct pca9685 *pca; unsigned int reg; int ret; - pca = devm_kzalloc(&client->dev, sizeof(*pca), GFP_KERNEL); - if (!pca) - return -ENOMEM; + /* Add an extra channel for ALL_LED */ + chip = devm_pwmchip_alloc(&client->dev, PCA9685_MAXCHAN + 1, sizeof(*pca)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pca = to_pca(chip); pca->regmap = devm_regmap_init_i2c(client, &pca9685_regmap_i2c_config); if (IS_ERR(pca->regmap)) { @@ -532,11 +539,11 @@ static int pca9685_pwm_probe(struct i2c_client *client) return ret; } - i2c_set_clientdata(client, pca); + i2c_set_clientdata(client, chip); mutex_init(&pca->lock); - ret = pca9685_read_reg(pca, PCA9685_MODE2, ®); + ret = pca9685_read_reg(chip, PCA9685_MODE2, ®); if (ret) return ret; @@ -550,34 +557,30 @@ static int pca9685_pwm_probe(struct i2c_client *client) else reg |= MODE2_OUTDRV; - ret = pca9685_write_reg(pca, PCA9685_MODE2, reg); + ret = pca9685_write_reg(chip, PCA9685_MODE2, reg); if (ret) return ret; /* Disable all LED ALLCALL and SUBx addresses to avoid bus collisions */ - pca9685_read_reg(pca, PCA9685_MODE1, ®); + pca9685_read_reg(chip, PCA9685_MODE1, ®); reg &= ~(MODE1_ALLCALL | MODE1_SUB1 | MODE1_SUB2 | MODE1_SUB3); - pca9685_write_reg(pca, PCA9685_MODE1, reg); + pca9685_write_reg(chip, PCA9685_MODE1, reg); /* Reset OFF/ON registers to POR default */ - pca9685_write_reg(pca, PCA9685_ALL_LED_OFF_L, 0); - pca9685_write_reg(pca, PCA9685_ALL_LED_OFF_H, LED_FULL); - pca9685_write_reg(pca, PCA9685_ALL_LED_ON_L, 0); - pca9685_write_reg(pca, PCA9685_ALL_LED_ON_H, LED_FULL); + pca9685_write_reg(chip, PCA9685_ALL_LED_OFF_L, 0); + pca9685_write_reg(chip, PCA9685_ALL_LED_OFF_H, LED_FULL); + pca9685_write_reg(chip, PCA9685_ALL_LED_ON_L, 0); + pca9685_write_reg(chip, PCA9685_ALL_LED_ON_H, LED_FULL); - pca->chip.ops = &pca9685_pwm_ops; - /* Add an extra channel for ALL_LED */ - pca->chip.npwm = PCA9685_MAXCHAN + 1; + chip->ops = &pca9685_pwm_ops; - pca->chip.dev = &client->dev; - - ret = pwmchip_add(&pca->chip); + ret = pwmchip_add(chip); if (ret < 0) return ret; - ret = pca9685_pwm_gpio_probe(pca); + ret = pca9685_pwm_gpio_probe(chip); if (ret < 0) { - pwmchip_remove(&pca->chip); + pwmchip_remove(chip); return ret; } @@ -588,11 +591,11 @@ static int pca9685_pwm_probe(struct i2c_client *client) * Although the chip comes out of power-up in the sleep state, * we force it to sleep in case it was woken up before */ - pca9685_set_sleep_mode(pca, true); + pca9685_set_sleep_mode(chip, true); pm_runtime_set_suspended(&client->dev); } else { /* Wake the chip up if runtime PM is disabled */ - pca9685_set_sleep_mode(pca, false); + pca9685_set_sleep_mode(chip, false); } return 0; @@ -600,13 +603,13 @@ static int pca9685_pwm_probe(struct i2c_client *client) static void pca9685_pwm_remove(struct i2c_client *client) { - struct pca9685 *pca = i2c_get_clientdata(client); + struct pwm_chip *chip = i2c_get_clientdata(client); - pwmchip_remove(&pca->chip); + pwmchip_remove(chip); if (!pm_runtime_enabled(&client->dev)) { /* Put chip in sleep state if runtime PM is disabled */ - pca9685_set_sleep_mode(pca, true); + pca9685_set_sleep_mode(chip, true); } pm_runtime_disable(&client->dev); @@ -615,18 +618,18 @@ static void pca9685_pwm_remove(struct i2c_client *client) static int __maybe_unused pca9685_pwm_runtime_suspend(struct device *dev) { struct i2c_client *client = to_i2c_client(dev); - struct pca9685 *pca = i2c_get_clientdata(client); + struct pwm_chip *chip = i2c_get_clientdata(client); - pca9685_set_sleep_mode(pca, true); + pca9685_set_sleep_mode(chip, true); return 0; } static int __maybe_unused pca9685_pwm_runtime_resume(struct device *dev) { struct i2c_client *client = to_i2c_client(dev); - struct pca9685 *pca = i2c_get_clientdata(client); + struct pwm_chip *chip = i2c_get_clientdata(client); - pca9685_set_sleep_mode(pca, false); + pca9685_set_sleep_mode(chip, false); return 0; } diff --git a/drivers/pwm/pwm-pxa.c b/drivers/pwm/pwm-pxa.c index 76685f926c75..bb7bb48b2e6d 100644 --- a/drivers/pwm/pwm-pxa.c +++ b/drivers/pwm/pwm-pxa.c @@ -49,7 +49,6 @@ MODULE_DEVICE_TABLE(platform, pwm_id_table); #define PWMDCR_FD (1 << 10) struct pxa_pwm_chip { - struct pwm_chip chip; struct device *dev; struct clk *clk; @@ -58,7 +57,7 @@ struct pxa_pwm_chip { static inline struct pxa_pwm_chip *to_pxa_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct pxa_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } /* @@ -159,6 +158,7 @@ MODULE_DEVICE_TABLE(of, pwm_of_match); static int pwm_probe(struct platform_device *pdev) { const struct platform_device_id *id = platform_get_device_id(pdev); + struct pwm_chip *chip; struct pxa_pwm_chip *pc; int ret = 0; @@ -168,28 +168,27 @@ static int pwm_probe(struct platform_device *pdev) if (id == NULL) return -EINVAL; - pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL); - if (pc == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, + (id->driver_data & HAS_SECONDARY_PWM) ? 2 : 1, + sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_pxa_pwm_chip(chip); pc->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(pc->clk)) return PTR_ERR(pc->clk); - pc->chip.dev = &pdev->dev; - pc->chip.ops = &pxa_pwm_ops; - pc->chip.npwm = (id->driver_data & HAS_SECONDARY_PWM) ? 2 : 1; + chip->ops = &pxa_pwm_ops; - if (IS_ENABLED(CONFIG_OF)) { - pc->chip.of_xlate = of_pwm_single_xlate; - pc->chip.of_pwm_n_cells = 1; - } + if (IS_ENABLED(CONFIG_OF)) + chip->of_xlate = of_pwm_single_xlate; pc->mmio_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->mmio_base)) return PTR_ERR(pc->mmio_base); - ret = devm_pwmchip_add(&pdev->dev, &pc->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) { dev_err(&pdev->dev, "pwmchip_add() failed: %d\n", ret); return ret; diff --git a/drivers/pwm/pwm-raspberrypi-poe.c b/drivers/pwm/pwm-raspberrypi-poe.c index 1ad814fdec6b..8921e7ea2cea 100644 --- a/drivers/pwm/pwm-raspberrypi-poe.c +++ b/drivers/pwm/pwm-raspberrypi-poe.c @@ -27,7 +27,6 @@ struct raspberrypi_pwm { struct rpi_firmware *firmware; - struct pwm_chip chip; unsigned int duty_cycle; }; @@ -40,7 +39,7 @@ struct raspberrypi_pwm_prop { static inline struct raspberrypi_pwm *raspberrypi_pwm_from_chip(struct pwm_chip *chip) { - return container_of(chip, struct raspberrypi_pwm, chip); + return pwmchip_get_drvdata(chip); } static int raspberrypi_pwm_set_property(struct rpi_firmware *firmware, @@ -122,7 +121,7 @@ static int raspberrypi_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, ret = raspberrypi_pwm_set_property(rpipwm->firmware, RPI_PWM_CUR_DUTY_REG, duty_cycle); if (ret) { - dev_err(chip->dev, "Failed to set duty cycle: %pe\n", + dev_err(pwmchip_parent(chip), "Failed to set duty cycle: %pe\n", ERR_PTR(ret)); return ret; } @@ -142,6 +141,7 @@ static int raspberrypi_pwm_probe(struct platform_device *pdev) struct device_node *firmware_node; struct device *dev = &pdev->dev; struct rpi_firmware *firmware; + struct pwm_chip *chip; struct raspberrypi_pwm *rpipwm; int ret; @@ -157,14 +157,14 @@ static int raspberrypi_pwm_probe(struct platform_device *pdev) return dev_err_probe(dev, -EPROBE_DEFER, "Failed to get firmware handle\n"); - rpipwm = devm_kzalloc(&pdev->dev, sizeof(*rpipwm), GFP_KERNEL); - if (!rpipwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, RASPBERRYPI_FIRMWARE_PWM_NUM, + sizeof(*rpipwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + rpipwm = raspberrypi_pwm_from_chip(chip); rpipwm->firmware = firmware; - rpipwm->chip.dev = dev; - rpipwm->chip.ops = &raspberrypi_pwm_ops; - rpipwm->chip.npwm = RASPBERRYPI_FIRMWARE_PWM_NUM; + chip->ops = &raspberrypi_pwm_ops; ret = raspberrypi_pwm_get_property(rpipwm->firmware, RPI_PWM_CUR_DUTY_REG, &rpipwm->duty_cycle); @@ -173,7 +173,7 @@ static int raspberrypi_pwm_probe(struct platform_device *pdev) return ret; } - return devm_pwmchip_add(dev, &rpipwm->chip); + return devm_pwmchip_add(dev, chip); } static const struct of_device_id raspberrypi_pwm_of_match[] = { diff --git a/drivers/pwm/pwm-rcar.c b/drivers/pwm/pwm-rcar.c index 13269f55fccf..4cfecd88ede0 100644 --- a/drivers/pwm/pwm-rcar.c +++ b/drivers/pwm/pwm-rcar.c @@ -38,14 +38,13 @@ #define RCAR_PWMCNT_PH0_SHIFT 0 struct rcar_pwm_chip { - struct pwm_chip chip; void __iomem *base; struct clk *clk; }; static inline struct rcar_pwm_chip *to_rcar_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct rcar_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static void rcar_pwm_write(struct rcar_pwm_chip *rp, u32 data, @@ -132,12 +131,12 @@ static int rcar_pwm_set_counter(struct rcar_pwm_chip *rp, int div, int duty_ns, static int rcar_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) { - return pm_runtime_get_sync(chip->dev); + return pm_runtime_get_sync(pwmchip_parent(chip)); } static void rcar_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) { - pm_runtime_put(chip->dev); + pm_runtime_put(pwmchip_parent(chip)); } static int rcar_pwm_enable(struct rcar_pwm_chip *rp) @@ -202,12 +201,14 @@ static const struct pwm_ops rcar_pwm_ops = { static int rcar_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct rcar_pwm_chip *rcar_pwm; int ret; - rcar_pwm = devm_kzalloc(&pdev->dev, sizeof(*rcar_pwm), GFP_KERNEL); - if (rcar_pwm == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*rcar_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + rcar_pwm = to_rcar_pwm_chip(chip); rcar_pwm->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(rcar_pwm->base)) @@ -219,15 +220,13 @@ static int rcar_pwm_probe(struct platform_device *pdev) return PTR_ERR(rcar_pwm->clk); } - platform_set_drvdata(pdev, rcar_pwm); + chip->ops = &rcar_pwm_ops; - rcar_pwm->chip.dev = &pdev->dev; - rcar_pwm->chip.ops = &rcar_pwm_ops; - rcar_pwm->chip.npwm = 1; + platform_set_drvdata(pdev, chip); pm_runtime_enable(&pdev->dev); - ret = pwmchip_add(&rcar_pwm->chip); + ret = pwmchip_add(chip); if (ret < 0) { dev_err(&pdev->dev, "failed to register PWM chip: %d\n", ret); pm_runtime_disable(&pdev->dev); @@ -239,9 +238,9 @@ static int rcar_pwm_probe(struct platform_device *pdev) static void rcar_pwm_remove(struct platform_device *pdev) { - struct rcar_pwm_chip *rcar_pwm = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); - pwmchip_remove(&rcar_pwm->chip); + pwmchip_remove(chip); pm_runtime_disable(&pdev->dev); } diff --git a/drivers/pwm/pwm-renesas-tpu.c b/drivers/pwm/pwm-renesas-tpu.c index 28265fdfc92a..2196080b4177 100644 --- a/drivers/pwm/pwm-renesas-tpu.c +++ b/drivers/pwm/pwm-renesas-tpu.c @@ -79,7 +79,6 @@ struct tpu_pwm_device { struct tpu_device { struct platform_device *pdev; - struct pwm_chip chip; spinlock_t lock; void __iomem *base; @@ -87,7 +86,10 @@ struct tpu_device { struct tpu_pwm_device tpd[TPU_CHANNEL_MAX]; }; -#define to_tpu_device(c) container_of(c, struct tpu_device, chip) +static inline struct tpu_device *to_tpu_device(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} static void tpu_pwm_write(struct tpu_pwm_device *tpd, int reg_nr, u16 value) { @@ -438,12 +440,14 @@ static const struct pwm_ops tpu_pwm_ops = { static int tpu_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct tpu_device *tpu; int ret; - tpu = devm_kzalloc(&pdev->dev, sizeof(*tpu), GFP_KERNEL); - if (tpu == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, TPU_CHANNEL_MAX, sizeof(*tpu)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + tpu = to_tpu_device(chip); spin_lock_init(&tpu->lock); tpu->pdev = pdev; @@ -460,15 +464,13 @@ static int tpu_probe(struct platform_device *pdev) /* Initialize and register the device. */ platform_set_drvdata(pdev, tpu); - tpu->chip.dev = &pdev->dev; - tpu->chip.ops = &tpu_pwm_ops; - tpu->chip.npwm = TPU_CHANNEL_MAX; + chip->ops = &tpu_pwm_ops; ret = devm_pm_runtime_enable(&pdev->dev); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "Failed to enable runtime PM\n"); - ret = devm_pwmchip_add(&pdev->dev, &tpu->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "Failed to register PWM chip\n"); diff --git a/drivers/pwm/pwm-rockchip.c b/drivers/pwm/pwm-rockchip.c index a7c647e37837..0fa7575dbb54 100644 --- a/drivers/pwm/pwm-rockchip.c +++ b/drivers/pwm/pwm-rockchip.c @@ -30,7 +30,6 @@ #define PWM_LP_DISABLE (0 << 8) struct rockchip_pwm_chip { - struct pwm_chip chip; struct clk *clk; struct clk *pclk; const struct rockchip_pwm_data *data; @@ -54,7 +53,7 @@ struct rockchip_pwm_data { static inline struct rockchip_pwm_chip *to_rockchip_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct rockchip_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static int rockchip_pwm_get_state(struct pwm_chip *chip, @@ -296,14 +295,16 @@ MODULE_DEVICE_TABLE(of, rockchip_pwm_dt_ids); static int rockchip_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct rockchip_pwm_chip *pc; u32 enable_conf, ctrl; bool enabled; int ret, count; - pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL); - if (!pc) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_rockchip_pwm_chip(chip); pc->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->base)) @@ -337,18 +338,16 @@ static int rockchip_pwm_probe(struct platform_device *pdev) goto err_clk; } - platform_set_drvdata(pdev, pc); + platform_set_drvdata(pdev, chip); pc->data = device_get_match_data(&pdev->dev); - pc->chip.dev = &pdev->dev; - pc->chip.ops = &rockchip_pwm_ops; - pc->chip.npwm = 1; + chip->ops = &rockchip_pwm_ops; enable_conf = pc->data->enable_conf; ctrl = readl_relaxed(pc->base + pc->data->regs.ctrl); enabled = (ctrl & enable_conf) == enable_conf; - ret = pwmchip_add(&pc->chip); + ret = pwmchip_add(chip); if (ret < 0) { dev_err_probe(&pdev->dev, ret, "pwmchip_add() failed\n"); goto err_pclk; @@ -372,9 +371,10 @@ err_clk: static void rockchip_pwm_remove(struct platform_device *pdev) { - struct rockchip_pwm_chip *pc = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct rockchip_pwm_chip *pc = to_rockchip_pwm_chip(chip); - pwmchip_remove(&pc->chip); + pwmchip_remove(chip); clk_unprepare(pc->pclk); clk_unprepare(pc->clk); diff --git a/drivers/pwm/pwm-rz-mtu3.c b/drivers/pwm/pwm-rz-mtu3.c index bdda315b3bd3..ab39bd37edaf 100644 --- a/drivers/pwm/pwm-rz-mtu3.c +++ b/drivers/pwm/pwm-rz-mtu3.c @@ -61,7 +61,6 @@ struct rz_mtu3_pwm_channel { /** * struct rz_mtu3_pwm_chip - MTU3 pwm private data * - * @chip: MTU3 pwm chip data * @clk: MTU3 module clock * @lock: Lock to prevent concurrent access for usage count * @rate: MTU3 clock rate @@ -72,7 +71,6 @@ struct rz_mtu3_pwm_channel { */ struct rz_mtu3_pwm_chip { - struct pwm_chip chip; struct clk *clk; struct mutex lock; unsigned long rate; @@ -92,7 +90,7 @@ static const struct rz_mtu3_channel_io_map channel_map[] = { static inline struct rz_mtu3_pwm_chip *to_rz_mtu3_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct rz_mtu3_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static void rz_mtu3_pwm_read_tgr_registers(struct rz_mtu3_pwm_channel *priv, @@ -211,15 +209,15 @@ static void rz_mtu3_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) mutex_unlock(&rz_mtu3_pwm->lock); } -static int rz_mtu3_pwm_enable(struct rz_mtu3_pwm_chip *rz_mtu3_pwm, - struct pwm_device *pwm) +static int rz_mtu3_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) { + struct rz_mtu3_pwm_chip *rz_mtu3_pwm = to_rz_mtu3_pwm_chip(chip); struct rz_mtu3_pwm_channel *priv; u32 ch; u8 val; int rc; - rc = pm_runtime_resume_and_get(rz_mtu3_pwm->chip.dev); + rc = pm_runtime_resume_and_get(pwmchip_parent(chip)); if (rc) return rc; @@ -243,9 +241,9 @@ static int rz_mtu3_pwm_enable(struct rz_mtu3_pwm_chip *rz_mtu3_pwm, return 0; } -static void rz_mtu3_pwm_disable(struct rz_mtu3_pwm_chip *rz_mtu3_pwm, - struct pwm_device *pwm) +static void rz_mtu3_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) { + struct rz_mtu3_pwm_chip *rz_mtu3_pwm = to_rz_mtu3_pwm_chip(chip); struct rz_mtu3_pwm_channel *priv; u32 ch; @@ -265,7 +263,7 @@ static void rz_mtu3_pwm_disable(struct rz_mtu3_pwm_chip *rz_mtu3_pwm, mutex_unlock(&rz_mtu3_pwm->lock); - pm_runtime_put_sync(rz_mtu3_pwm->chip.dev); + pm_runtime_put_sync(pwmchip_parent(chip)); } static int rz_mtu3_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, @@ -274,7 +272,7 @@ static int rz_mtu3_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, struct rz_mtu3_pwm_chip *rz_mtu3_pwm = to_rz_mtu3_pwm_chip(chip); int rc; - rc = pm_runtime_resume_and_get(chip->dev); + rc = pm_runtime_resume_and_get(pwmchip_parent(chip)); if (rc) return rc; @@ -307,7 +305,7 @@ static int rz_mtu3_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, } state->polarity = PWM_POLARITY_NORMAL; - pm_runtime_put(chip->dev); + pm_runtime_put(pwmchip_parent(chip)); return 0; } @@ -362,7 +360,7 @@ static int rz_mtu3_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if (!pwm->state.enabled) { int rc; - rc = pm_runtime_resume_and_get(chip->dev); + rc = pm_runtime_resume_and_get(pwmchip_parent(chip)); if (rc) return rc; } @@ -399,7 +397,7 @@ static int rz_mtu3_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, /* If the PWM is not enabled, turn the clock off again to save power. */ if (!pwm->state.enabled) - pm_runtime_put(chip->dev); + pm_runtime_put(pwmchip_parent(chip)); return 0; } @@ -416,7 +414,7 @@ static int rz_mtu3_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (!state->enabled) { if (enabled) - rz_mtu3_pwm_disable(rz_mtu3_pwm, pwm); + rz_mtu3_pwm_disable(chip, pwm); return 0; } @@ -428,7 +426,7 @@ static int rz_mtu3_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, return ret; if (!enabled) - ret = rz_mtu3_pwm_enable(rz_mtu3_pwm, pwm); + ret = rz_mtu3_pwm_enable(chip, pwm); return ret; } @@ -442,7 +440,8 @@ static const struct pwm_ops rz_mtu3_pwm_ops = { static int rz_mtu3_pwm_pm_runtime_suspend(struct device *dev) { - struct rz_mtu3_pwm_chip *rz_mtu3_pwm = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct rz_mtu3_pwm_chip *rz_mtu3_pwm = to_rz_mtu3_pwm_chip(chip); clk_disable_unprepare(rz_mtu3_pwm->clk); @@ -451,7 +450,8 @@ static int rz_mtu3_pwm_pm_runtime_suspend(struct device *dev) static int rz_mtu3_pwm_pm_runtime_resume(struct device *dev) { - struct rz_mtu3_pwm_chip *rz_mtu3_pwm = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct rz_mtu3_pwm_chip *rz_mtu3_pwm = to_rz_mtu3_pwm_chip(chip); return clk_prepare_enable(rz_mtu3_pwm->clk); } @@ -462,24 +462,28 @@ static DEFINE_RUNTIME_DEV_PM_OPS(rz_mtu3_pwm_pm_ops, static void rz_mtu3_pwm_pm_disable(void *data) { - struct rz_mtu3_pwm_chip *rz_mtu3_pwm = data; + struct pwm_chip *chip = data; + struct rz_mtu3_pwm_chip *rz_mtu3_pwm = to_rz_mtu3_pwm_chip(chip); clk_rate_exclusive_put(rz_mtu3_pwm->clk); - pm_runtime_disable(rz_mtu3_pwm->chip.dev); - pm_runtime_set_suspended(rz_mtu3_pwm->chip.dev); + pm_runtime_disable(pwmchip_parent(chip)); + pm_runtime_set_suspended(pwmchip_parent(chip)); } static int rz_mtu3_pwm_probe(struct platform_device *pdev) { struct rz_mtu3 *parent_ddata = dev_get_drvdata(pdev->dev.parent); struct rz_mtu3_pwm_chip *rz_mtu3_pwm; + struct pwm_chip *chip; struct device *dev = &pdev->dev; unsigned int i, j = 0; int ret; - rz_mtu3_pwm = devm_kzalloc(&pdev->dev, sizeof(*rz_mtu3_pwm), GFP_KERNEL); - if (!rz_mtu3_pwm) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, RZ_MTU3_MAX_PWM_CHANNELS, + sizeof(*rz_mtu3_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + rz_mtu3_pwm = to_rz_mtu3_pwm_chip(chip); rz_mtu3_pwm->clk = parent_ddata->clk; @@ -494,7 +498,7 @@ static int rz_mtu3_pwm_probe(struct platform_device *pdev) } mutex_init(&rz_mtu3_pwm->lock); - platform_set_drvdata(pdev, rz_mtu3_pwm); + platform_set_drvdata(pdev, chip); ret = clk_prepare_enable(rz_mtu3_pwm->clk); if (ret) return dev_err_probe(dev, ret, "Clock enable failed\n"); @@ -514,15 +518,13 @@ static int rz_mtu3_pwm_probe(struct platform_device *pdev) pm_runtime_set_active(&pdev->dev); pm_runtime_enable(&pdev->dev); - rz_mtu3_pwm->chip.dev = &pdev->dev; ret = devm_add_action_or_reset(&pdev->dev, rz_mtu3_pwm_pm_disable, - rz_mtu3_pwm); + chip); if (ret < 0) return ret; - rz_mtu3_pwm->chip.ops = &rz_mtu3_pwm_ops; - rz_mtu3_pwm->chip.npwm = RZ_MTU3_MAX_PWM_CHANNELS; - ret = devm_pwmchip_add(&pdev->dev, &rz_mtu3_pwm->chip); + chip->ops = &rz_mtu3_pwm_ops; + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret) return dev_err_probe(&pdev->dev, ret, "failed to add PWM chip\n"); diff --git a/drivers/pwm/pwm-samsung.c b/drivers/pwm/pwm-samsung.c index 6e77302f7368..efb60c9f0cb3 100644 --- a/drivers/pwm/pwm-samsung.c +++ b/drivers/pwm/pwm-samsung.c @@ -69,7 +69,6 @@ struct samsung_pwm_channel { /** * struct samsung_pwm_chip - private data of PWM chip - * @chip: generic PWM chip * @variant: local copy of hardware variant data * @inverter_mask: inverter status for all channels - one bit per channel * @disabled_mask: disabled status for all channels - one bit per channel @@ -80,7 +79,6 @@ struct samsung_pwm_channel { * @channel: per channel driver data */ struct samsung_pwm_chip { - struct pwm_chip chip; struct samsung_pwm_variant variant; u8 inverter_mask; u8 disabled_mask; @@ -110,7 +108,7 @@ static DEFINE_SPINLOCK(samsung_pwm_lock); static inline struct samsung_pwm_chip *to_samsung_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct samsung_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static inline unsigned int to_tcon_channel(unsigned int channel) @@ -181,9 +179,10 @@ static unsigned long pwm_samsung_get_tin_rate(struct samsung_pwm_chip *our_chip, return rate / (reg + 1); } -static unsigned long pwm_samsung_calc_tin(struct samsung_pwm_chip *our_chip, +static unsigned long pwm_samsung_calc_tin(struct pwm_chip *chip, unsigned int chan, unsigned long freq) { + struct samsung_pwm_chip *our_chip = to_samsung_pwm_chip(chip); struct samsung_pwm_variant *variant = &our_chip->variant; unsigned long rate; struct clk *clk; @@ -197,12 +196,12 @@ static unsigned long pwm_samsung_calc_tin(struct samsung_pwm_chip *our_chip, return rate; } - dev_warn(our_chip->chip.dev, + dev_warn(pwmchip_parent(chip), "tclk of PWM %d is inoperational, using tdiv\n", chan); } rate = pwm_samsung_get_tin_rate(our_chip, chan); - dev_dbg(our_chip->chip.dev, "tin parent at %lu\n", rate); + dev_dbg(pwmchip_parent(chip), "tin parent at %lu\n", rate); /* * Compare minimum PWM frequency that can be achieved with possible @@ -232,7 +231,7 @@ static int pwm_samsung_request(struct pwm_chip *chip, struct pwm_device *pwm) struct samsung_pwm_chip *our_chip = to_samsung_pwm_chip(chip); if (!(our_chip->variant.output_mask & BIT(pwm->hwpwm))) { - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), "tried to request PWM channel %d without output\n", pwm->hwpwm); return -EINVAL; @@ -326,12 +325,12 @@ static int __pwm_samsung_config(struct pwm_chip *chip, struct pwm_device *pwm, period = NSEC_PER_SEC / period_ns; - dev_dbg(our_chip->chip.dev, "duty_ns=%d, period_ns=%d (%u)\n", + dev_dbg(pwmchip_parent(chip), "duty_ns=%d, period_ns=%d (%u)\n", duty_ns, period_ns, period); - tin_rate = pwm_samsung_calc_tin(our_chip, pwm->hwpwm, period); + tin_rate = pwm_samsung_calc_tin(chip, pwm->hwpwm, period); - dev_dbg(our_chip->chip.dev, "tin_rate=%lu\n", tin_rate); + dev_dbg(pwmchip_parent(chip), "tin_rate=%lu\n", tin_rate); tin_ns = NSEC_PER_SEC / tin_rate; tcnt = period_ns / tin_ns; @@ -355,8 +354,7 @@ static int __pwm_samsung_config(struct pwm_chip *chip, struct pwm_device *pwm, /* -1UL will give 100% duty. */ --tcmp; - dev_dbg(our_chip->chip.dev, - "tin_ns=%u, tcmp=%u/%u\n", tin_ns, tcmp, tcnt); + dev_dbg(pwmchip_parent(chip), "tin_ns=%u, tcmp=%u/%u\n", tin_ns, tcmp, tcnt); /* Update PWM registers. */ writel(tcnt, our_chip->base + REG_TCNTB(pwm->hwpwm)); @@ -368,7 +366,7 @@ static int __pwm_samsung_config(struct pwm_chip *chip, struct pwm_device *pwm, * shortly afer this update (before it autoreloaded the new values). */ if (oldtcmp == (u32) -1) { - dev_dbg(our_chip->chip.dev, "Forcing manual update"); + dev_dbg(pwmchip_parent(chip), "Forcing manual update"); pwm_samsung_manual_update(our_chip, pwm); } @@ -507,9 +505,10 @@ static const struct of_device_id samsung_pwm_matches[] = { }; MODULE_DEVICE_TABLE(of, samsung_pwm_matches); -static int pwm_samsung_parse_dt(struct samsung_pwm_chip *our_chip) +static int pwm_samsung_parse_dt(struct pwm_chip *chip) { - struct device_node *np = our_chip->chip.dev->of_node; + struct samsung_pwm_chip *our_chip = to_samsung_pwm_chip(chip); + struct device_node *np = pwmchip_parent(chip)->of_node; const struct of_device_id *match; struct property *prop; const __be32 *cur; @@ -523,7 +522,7 @@ static int pwm_samsung_parse_dt(struct samsung_pwm_chip *our_chip) of_property_for_each_u32(np, "samsung,pwm-outputs", prop, cur, val) { if (val >= SAMSUNG_PWM_NUM) { - dev_err(our_chip->chip.dev, + dev_err(pwmchip_parent(chip), "%s: invalid channel index in samsung,pwm-outputs property\n", __func__); continue; @@ -534,7 +533,7 @@ static int pwm_samsung_parse_dt(struct samsung_pwm_chip *our_chip) return 0; } #else -static int pwm_samsung_parse_dt(struct samsung_pwm_chip *our_chip) +static int pwm_samsung_parse_dt(struct pwm_chip *chip) { return -ENODEV; } @@ -544,27 +543,26 @@ static int pwm_samsung_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct samsung_pwm_chip *our_chip; + struct pwm_chip *chip; unsigned int chan; int ret; - our_chip = devm_kzalloc(&pdev->dev, sizeof(*our_chip), GFP_KERNEL); - if (our_chip == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, SAMSUNG_PWM_NUM, sizeof(*our_chip)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + our_chip = to_samsung_pwm_chip(chip); - our_chip->chip.dev = &pdev->dev; - our_chip->chip.ops = &pwm_samsung_ops; - our_chip->chip.npwm = SAMSUNG_PWM_NUM; + chip->ops = &pwm_samsung_ops; our_chip->inverter_mask = BIT(SAMSUNG_PWM_NUM) - 1; if (IS_ENABLED(CONFIG_OF) && pdev->dev.of_node) { - ret = pwm_samsung_parse_dt(our_chip); + ret = pwm_samsung_parse_dt(chip); if (ret) return ret; } else { - if (!pdev->dev.platform_data) { - dev_err(&pdev->dev, "no platform data specified\n"); - return -EINVAL; - } + if (!pdev->dev.platform_data) + return dev_err_probe(&pdev->dev, -EINVAL, + "no platform data specified\n"); memcpy(&our_chip->variant, pdev->dev.platform_data, sizeof(our_chip->variant)); @@ -574,17 +572,10 @@ static int pwm_samsung_probe(struct platform_device *pdev) if (IS_ERR(our_chip->base)) return PTR_ERR(our_chip->base); - our_chip->base_clk = devm_clk_get(&pdev->dev, "timers"); - if (IS_ERR(our_chip->base_clk)) { - dev_err(dev, "failed to get timer base clk\n"); - return PTR_ERR(our_chip->base_clk); - } - - ret = clk_prepare_enable(our_chip->base_clk); - if (ret < 0) { - dev_err(dev, "failed to enable base clock\n"); - return ret; - } + our_chip->base_clk = devm_clk_get_enabled(&pdev->dev, "timers"); + if (IS_ERR(our_chip->base_clk)) + return dev_err_probe(dev, PTR_ERR(our_chip->base_clk), + "failed to get timer base clk\n"); for (chan = 0; chan < SAMSUNG_PWM_NUM; ++chan) if (our_chip->variant.output_mask & BIT(chan)) @@ -594,14 +585,11 @@ static int pwm_samsung_probe(struct platform_device *pdev) our_chip->tclk0 = devm_clk_get(&pdev->dev, "pwm-tclk0"); our_chip->tclk1 = devm_clk_get(&pdev->dev, "pwm-tclk1"); - platform_set_drvdata(pdev, our_chip); + platform_set_drvdata(pdev, chip); - ret = pwmchip_add(&our_chip->chip); - if (ret < 0) { - dev_err(dev, "failed to register PWM chip\n"); - clk_disable_unprepare(our_chip->base_clk); - return ret; - } + ret = devm_pwmchip_add(&pdev->dev, chip); + if (ret < 0) + return dev_err_probe(dev, ret, "failed to register PWM chip\n"); dev_dbg(dev, "base_clk at %lu, tclk0 at %lu, tclk1 at %lu\n", clk_get_rate(our_chip->base_clk), @@ -611,19 +599,10 @@ static int pwm_samsung_probe(struct platform_device *pdev) return 0; } -static void pwm_samsung_remove(struct platform_device *pdev) -{ - struct samsung_pwm_chip *our_chip = platform_get_drvdata(pdev); - - pwmchip_remove(&our_chip->chip); - - clk_disable_unprepare(our_chip->base_clk); -} - static int pwm_samsung_resume(struct device *dev) { - struct samsung_pwm_chip *our_chip = dev_get_drvdata(dev); - struct pwm_chip *chip = &our_chip->chip; + struct pwm_chip *chip = dev_get_drvdata(dev); + struct samsung_pwm_chip *our_chip = to_samsung_pwm_chip(chip); unsigned int i; for (i = 0; i < SAMSUNG_PWM_NUM; i++) { @@ -662,7 +641,6 @@ static struct platform_driver pwm_samsung_driver = { .of_match_table = of_match_ptr(samsung_pwm_matches), }, .probe = pwm_samsung_probe, - .remove_new = pwm_samsung_remove, }; module_platform_driver(pwm_samsung_driver); diff --git a/drivers/pwm/pwm-sifive.c b/drivers/pwm/pwm-sifive.c index 089e50bdbbf0..ed7957cc51fd 100644 --- a/drivers/pwm/pwm-sifive.c +++ b/drivers/pwm/pwm-sifive.c @@ -41,7 +41,7 @@ #define PWM_SIFIVE_DEFAULT_PERIOD 10000000 struct pwm_sifive_ddata { - struct pwm_chip chip; + struct device *parent; struct mutex lock; /* lock to protect user_count and approx_period */ struct notifier_block notifier; struct clk *clk; @@ -54,7 +54,7 @@ struct pwm_sifive_ddata { static inline struct pwm_sifive_ddata *pwm_sifive_chip_to_ddata(struct pwm_chip *chip) { - return container_of(chip, struct pwm_sifive_ddata, chip); + return pwmchip_get_drvdata(chip); } static int pwm_sifive_request(struct pwm_chip *chip, struct pwm_device *pwm) @@ -102,7 +102,7 @@ static void pwm_sifive_update_clock(struct pwm_sifive_ddata *ddata, /* As scale <= 15 the shift operation cannot overflow. */ num = (unsigned long long)NSEC_PER_SEC << (PWM_SIFIVE_CMPWIDTH + scale); ddata->real_period = div64_ul(num, rate); - dev_dbg(ddata->chip.dev, + dev_dbg(ddata->parent, "New real_period = %u ns\n", ddata->real_period); } @@ -185,7 +185,7 @@ static int pwm_sifive_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (!enabled) { ret = clk_enable(ddata->clk); if (ret) { - dev_err(ddata->chip.dev, "Enable clk failed\n"); + dev_err(pwmchip_parent(chip), "Enable clk failed\n"); return ret; } } @@ -230,15 +230,14 @@ static int pwm_sifive_probe(struct platform_device *pdev) u32 val; unsigned int enabled_pwms = 0, enabled_clks = 1; - ddata = devm_kzalloc(dev, sizeof(*ddata), GFP_KERNEL); - if (!ddata) - return -ENOMEM; + chip = devm_pwmchip_alloc(dev, 4, sizeof(*ddata)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + ddata = pwm_sifive_chip_to_ddata(chip); + ddata->parent = dev; mutex_init(&ddata->lock); - chip = &ddata->chip; - chip->dev = dev; chip->ops = &pwm_sifive_ops; - chip->npwm = 4; ddata->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(ddata->regs)) @@ -296,7 +295,7 @@ static int pwm_sifive_probe(struct platform_device *pdev) goto unregister_clk; } - platform_set_drvdata(pdev, ddata); + platform_set_drvdata(pdev, chip); dev_dbg(dev, "SiFive PWM chip registered %d PWMs\n", chip->npwm); return 0; @@ -314,15 +313,16 @@ disable_clk: static void pwm_sifive_remove(struct platform_device *dev) { - struct pwm_sifive_ddata *ddata = platform_get_drvdata(dev); + struct pwm_chip *chip = platform_get_drvdata(dev); + struct pwm_sifive_ddata *ddata = pwm_sifive_chip_to_ddata(chip); struct pwm_device *pwm; int ch; - pwmchip_remove(&ddata->chip); + pwmchip_remove(chip); clk_notifier_unregister(ddata->clk, &ddata->notifier); - for (ch = 0; ch < ddata->chip.npwm; ch++) { - pwm = &ddata->chip.pwms[ch]; + for (ch = 0; ch < chip->npwm; ch++) { + pwm = &chip->pwms[ch]; if (pwm->state.enabled) clk_disable(ddata->clk); } diff --git a/drivers/pwm/pwm-sl28cpld.c b/drivers/pwm/pwm-sl28cpld.c index 88b01ff9e460..934378d6a002 100644 --- a/drivers/pwm/pwm-sl28cpld.c +++ b/drivers/pwm/pwm-sl28cpld.c @@ -81,14 +81,13 @@ regmap_write((priv)->regmap, (priv)->offset + (reg), (val)) struct sl28cpld_pwm { - struct pwm_chip chip; struct regmap *regmap; u32 offset; }; static inline struct sl28cpld_pwm *sl28cpld_pwm_from_chip(struct pwm_chip *chip) { - return container_of(chip, struct sl28cpld_pwm, chip); + return pwmchip_get_drvdata(chip); } static int sl28cpld_pwm_get_state(struct pwm_chip *chip, @@ -213,9 +212,10 @@ static int sl28cpld_pwm_probe(struct platform_device *pdev) return -ENODEV; } - priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*priv)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + priv = sl28cpld_pwm_from_chip(chip); priv->regmap = dev_get_regmap(pdev->dev.parent, NULL); if (!priv->regmap) { @@ -231,10 +231,7 @@ static int sl28cpld_pwm_probe(struct platform_device *pdev) } /* Initialize the pwm_chip structure */ - chip = &priv->chip; - chip->dev = &pdev->dev; chip->ops = &sl28cpld_pwm_ops; - chip->npwm = 1; ret = devm_pwmchip_add(&pdev->dev, chip); if (ret) { diff --git a/drivers/pwm/pwm-spear.c b/drivers/pwm/pwm-spear.c index ff991319feef..6c6f3b38c835 100644 --- a/drivers/pwm/pwm-spear.c +++ b/drivers/pwm/pwm-spear.c @@ -48,17 +48,15 @@ * * @mmio_base: base address of pwm chip * @clk: pointer to clk structure of pwm chip - * @chip: linux pwm chip representation */ struct spear_pwm_chip { void __iomem *mmio_base; struct clk *clk; - struct pwm_chip chip; }; static inline struct spear_pwm_chip *to_spear_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct spear_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static inline u32 spear_pwm_readl(struct spear_pwm_chip *chip, unsigned int num, @@ -194,13 +192,15 @@ static const struct pwm_ops spear_pwm_ops = { static int spear_pwm_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; + struct pwm_chip *chip; struct spear_pwm_chip *pc; int ret; u32 val; - pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL); - if (!pc) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, NUM_PWM, sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_spear_pwm_chip(chip); pc->mmio_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->mmio_base)) @@ -211,9 +211,7 @@ static int spear_pwm_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(pc->clk), "Failed to get clock\n"); - pc->chip.dev = &pdev->dev; - pc->chip.ops = &spear_pwm_ops; - pc->chip.npwm = NUM_PWM; + chip->ops = &spear_pwm_ops; if (of_device_is_compatible(np, "st,spear1340-pwm")) { ret = clk_enable(pc->clk); @@ -232,7 +230,7 @@ static int spear_pwm_probe(struct platform_device *pdev) clk_disable(pc->clk); } - ret = devm_pwmchip_add(&pdev->dev, &pc->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "pwmchip_add() failed\n"); diff --git a/drivers/pwm/pwm-sprd.c b/drivers/pwm/pwm-sprd.c index 77939e161006..4c76ca5e4cdd 100644 --- a/drivers/pwm/pwm-sprd.c +++ b/drivers/pwm/pwm-sprd.c @@ -34,15 +34,12 @@ struct sprd_pwm_chn { struct sprd_pwm_chip { void __iomem *base; - struct device *dev; - struct pwm_chip chip; - int num_pwms; struct sprd_pwm_chn chn[SPRD_PWM_CHN_NUM]; }; static inline struct sprd_pwm_chip* sprd_pwm_from_chip(struct pwm_chip *chip) { - return container_of(chip, struct sprd_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } /* @@ -86,7 +83,7 @@ static int sprd_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, */ ret = clk_bulk_prepare_enable(SPRD_PWM_CHN_CLKS_NUM, chn->clks); if (ret) { - dev_err(spc->dev, "failed to enable pwm%u clocks\n", + dev_err(pwmchip_parent(chip), "failed to enable pwm%u clocks\n", pwm->hwpwm); return ret; } @@ -183,7 +180,7 @@ static int sprd_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, ret = clk_bulk_prepare_enable(SPRD_PWM_CHN_CLKS_NUM, chn->clks); if (ret) { - dev_err(spc->dev, + dev_err(pwmchip_parent(chip), "failed to enable pwm%u clocks\n", pwm->hwpwm); return ret; @@ -215,65 +212,64 @@ static const struct pwm_ops sprd_pwm_ops = { .get_state = sprd_pwm_get_state, }; -static int sprd_pwm_clk_init(struct sprd_pwm_chip *spc) +static int sprd_pwm_clk_init(struct device *dev, + struct sprd_pwm_chn chn[SPRD_PWM_CHN_NUM]) { struct clk *clk_pwm; int ret, i; for (i = 0; i < SPRD_PWM_CHN_NUM; i++) { - struct sprd_pwm_chn *chn = &spc->chn[i]; int j; for (j = 0; j < SPRD_PWM_CHN_CLKS_NUM; ++j) - chn->clks[j].id = + chn[i].clks[j].id = sprd_pwm_clks[i * SPRD_PWM_CHN_CLKS_NUM + j]; - ret = devm_clk_bulk_get(spc->dev, SPRD_PWM_CHN_CLKS_NUM, - chn->clks); + ret = devm_clk_bulk_get(dev, SPRD_PWM_CHN_CLKS_NUM, + chn[i].clks); if (ret) { if (ret == -ENOENT) break; - return dev_err_probe(spc->dev, ret, + return dev_err_probe(dev, ret, "failed to get channel clocks\n"); } - clk_pwm = chn->clks[SPRD_PWM_CHN_OUTPUT_CLK].clk; - chn->clk_rate = clk_get_rate(clk_pwm); + clk_pwm = chn[i].clks[SPRD_PWM_CHN_OUTPUT_CLK].clk; + chn[i].clk_rate = clk_get_rate(clk_pwm); } if (!i) - return dev_err_probe(spc->dev, -ENODEV, "no available PWM channels\n"); + return dev_err_probe(dev, -ENODEV, "no available PWM channels\n"); - spc->num_pwms = i; - - return 0; + return i; } static int sprd_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct sprd_pwm_chip *spc; - int ret; + struct sprd_pwm_chn chn[SPRD_PWM_CHN_NUM]; + int ret, npwm; - spc = devm_kzalloc(&pdev->dev, sizeof(*spc), GFP_KERNEL); - if (!spc) - return -ENOMEM; + npwm = sprd_pwm_clk_init(&pdev->dev, chn); + if (npwm < 0) + return npwm; + + chip = devm_pwmchip_alloc(&pdev->dev, npwm, sizeof(*spc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + spc = sprd_pwm_from_chip(chip); spc->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(spc->base)) return PTR_ERR(spc->base); - spc->dev = &pdev->dev; + memcpy(spc->chn, chn, sizeof(chn)); - ret = sprd_pwm_clk_init(spc); - if (ret) - return ret; + chip->ops = &sprd_pwm_ops; - spc->chip.dev = &pdev->dev; - spc->chip.ops = &sprd_pwm_ops; - spc->chip.npwm = spc->num_pwms; - - ret = devm_pwmchip_add(&pdev->dev, &spc->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret) dev_err(&pdev->dev, "failed to add PWM chip\n"); diff --git a/drivers/pwm/pwm-sti.c b/drivers/pwm/pwm-sti.c index 6cf55cf34d39..39d80da0e14a 100644 --- a/drivers/pwm/pwm-sti.c +++ b/drivers/pwm/pwm-sti.c @@ -94,7 +94,6 @@ struct sti_pwm_chip { struct regmap_field *pwm_cpt_en; struct regmap_field *pwm_cpt_int_en; struct regmap_field *pwm_cpt_int_stat; - struct pwm_chip chip; struct pwm_device *cur; unsigned long configured; unsigned int en_count; @@ -114,7 +113,7 @@ static const struct reg_field sti_pwm_regfields[MAX_REGFIELDS] = { static inline struct sti_pwm_chip *to_sti_pwmchip(struct pwm_chip *chip) { - return container_of(chip, struct sti_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } /* @@ -395,8 +394,17 @@ out: static int sti_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { + struct sti_pwm_chip *pc = to_sti_pwmchip(chip); + struct sti_pwm_compat_data *cdata = pc->cdata; + struct device *dev = pc->dev; int err; + if (pwm->hwpwm >= cdata->pwm_num_devs) { + dev_err(dev, "device %u is not valid for pwm mode\n", + pwm->hwpwm); + return -EINVAL; + } + if (state->polarity != PWM_POLARITY_NORMAL) return -EINVAL; @@ -498,23 +506,7 @@ static int sti_pwm_probe_dt(struct sti_pwm_chip *pc) { struct device *dev = pc->dev; const struct reg_field *reg_fields; - struct device_node *np = dev->of_node; struct sti_pwm_compat_data *cdata = pc->cdata; - u32 num_devs; - int ret; - - ret = of_property_read_u32(np, "st,pwm-num-chan", &num_devs); - if (!ret) - cdata->pwm_num_devs = num_devs; - - ret = of_property_read_u32(np, "st,capture-num-chan", &num_devs); - if (!ret) - cdata->cpt_num_devs = num_devs; - - if (!cdata->pwm_num_devs && !cdata->cpt_num_devs) { - dev_err(dev, "No channels configured\n"); - return -EINVAL; - } reg_fields = cdata->reg_fields; @@ -560,14 +552,33 @@ static const struct regmap_config sti_pwm_regmap_config = { static int sti_pwm_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + u32 num_devs; + unsigned int pwm_num_devs = 0; + unsigned int cpt_num_devs = 0; struct sti_pwm_compat_data *cdata; + struct pwm_chip *chip; struct sti_pwm_chip *pc; unsigned int i; int irq, ret; - pc = devm_kzalloc(dev, sizeof(*pc), GFP_KERNEL); - if (!pc) - return -ENOMEM; + ret = of_property_read_u32(np, "st,pwm-num-chan", &num_devs); + if (!ret) + pwm_num_devs = num_devs; + + ret = of_property_read_u32(np, "st,capture-num-chan", &num_devs); + if (!ret) + cpt_num_devs = num_devs; + + if (!pwm_num_devs && !cpt_num_devs) { + dev_err(dev, "No channels configured\n"); + return -EINVAL; + } + + chip = devm_pwmchip_alloc(dev, max(pwm_num_devs, cpt_num_devs), sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_sti_pwmchip(chip); cdata = devm_kzalloc(dev, sizeof(*cdata), GFP_KERNEL); if (!cdata) @@ -600,8 +611,8 @@ static int sti_pwm_probe(struct platform_device *pdev) cdata->reg_fields = sti_pwm_regfields; cdata->max_prescale = 0xff; cdata->max_pwm_cnt = 255; - cdata->pwm_num_devs = 0; - cdata->cpt_num_devs = 0; + cdata->pwm_num_devs = pwm_num_devs; + cdata->cpt_num_devs = cpt_num_devs; pc->cdata = cdata; pc->dev = dev; @@ -644,9 +655,7 @@ static int sti_pwm_probe(struct platform_device *pdev) return -ENOMEM; } - pc->chip.dev = dev; - pc->chip.ops = &sti_pwm_ops; - pc->chip.npwm = pc->cdata->pwm_num_devs; + chip->ops = &sti_pwm_ops; for (i = 0; i < cdata->cpt_num_devs; i++) { struct sti_cpt_ddata *ddata = &cdata->ddata[i]; @@ -655,23 +664,24 @@ static int sti_pwm_probe(struct platform_device *pdev) mutex_init(&ddata->lock); } - ret = pwmchip_add(&pc->chip); + ret = pwmchip_add(chip); if (ret < 0) { clk_unprepare(pc->pwm_clk); clk_unprepare(pc->cpt_clk); return ret; } - platform_set_drvdata(pdev, pc); + platform_set_drvdata(pdev, chip); return 0; } static void sti_pwm_remove(struct platform_device *pdev) { - struct sti_pwm_chip *pc = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct sti_pwm_chip *pc = to_sti_pwmchip(chip); - pwmchip_remove(&pc->chip); + pwmchip_remove(chip); clk_unprepare(pc->pwm_clk); clk_unprepare(pc->cpt_clk); diff --git a/drivers/pwm/pwm-stm32-lp.c b/drivers/pwm/pwm-stm32-lp.c index 439068f3eca1..989731256f50 100644 --- a/drivers/pwm/pwm-stm32-lp.c +++ b/drivers/pwm/pwm-stm32-lp.c @@ -18,14 +18,13 @@ #include struct stm32_pwm_lp { - struct pwm_chip chip; struct clk *clk; struct regmap *regmap; }; static inline struct stm32_pwm_lp *to_stm32_pwm_lp(struct pwm_chip *chip) { - return container_of(chip, struct stm32_pwm_lp, chip); + return pwmchip_get_drvdata(chip); } /* STM32 Low-Power Timer is preceded by a configurable power-of-2 prescaler */ @@ -61,7 +60,7 @@ static int stm32_pwm_lp_apply(struct pwm_chip *chip, struct pwm_device *pwm, do_div(div, NSEC_PER_SEC); if (!div) { /* Clock is too slow to achieve requested period. */ - dev_dbg(priv->chip.dev, "Can't reach %llu ns\n", state->period); + dev_dbg(pwmchip_parent(chip), "Can't reach %llu ns\n", state->period); return -EINVAL; } @@ -69,7 +68,7 @@ static int stm32_pwm_lp_apply(struct pwm_chip *chip, struct pwm_device *pwm, while (div > STM32_LPTIM_MAX_ARR) { presc++; if ((1 << presc) > STM32_LPTIM_MAX_PRESCALER) { - dev_err(priv->chip.dev, "max prescaler exceeded\n"); + dev_err(pwmchip_parent(chip), "max prescaler exceeded\n"); return -EINVAL; } div = prd >> presc; @@ -130,7 +129,7 @@ static int stm32_pwm_lp_apply(struct pwm_chip *chip, struct pwm_device *pwm, (val & STM32_LPTIM_CMPOK_ARROK) == STM32_LPTIM_CMPOK_ARROK, 100, 1000); if (ret) { - dev_err(priv->chip.dev, "ARR/CMP registers write issue\n"); + dev_err(pwmchip_parent(chip), "ARR/CMP registers write issue\n"); goto err; } ret = regmap_write(priv->regmap, STM32_LPTIM_ICR, @@ -197,36 +196,36 @@ static int stm32_pwm_lp_probe(struct platform_device *pdev) { struct stm32_lptimer *ddata = dev_get_drvdata(pdev->dev.parent); struct stm32_pwm_lp *priv; + struct pwm_chip *chip; int ret; - priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*priv)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + priv = to_stm32_pwm_lp(chip); priv->regmap = ddata->regmap; priv->clk = ddata->clk; - priv->chip.dev = &pdev->dev; - priv->chip.ops = &stm32_pwm_lp_ops; - priv->chip.npwm = 1; + chip->ops = &stm32_pwm_lp_ops; - ret = devm_pwmchip_add(&pdev->dev, &priv->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return ret; - platform_set_drvdata(pdev, priv); + platform_set_drvdata(pdev, chip); return 0; } static int stm32_pwm_lp_suspend(struct device *dev) { - struct stm32_pwm_lp *priv = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); struct pwm_state state; - pwm_get_state(&priv->chip.pwms[0], &state); + pwm_get_state(&chip->pwms[0], &state); if (state.enabled) { dev_err(dev, "The consumer didn't stop us (%s)\n", - priv->chip.pwms[0].label); + chip->pwms[0].label); return -EBUSY; } diff --git a/drivers/pwm/pwm-stm32.c b/drivers/pwm/pwm-stm32.c index 5f10cba492ec..0c028d17c075 100644 --- a/drivers/pwm/pwm-stm32.c +++ b/drivers/pwm/pwm-stm32.c @@ -27,7 +27,6 @@ struct stm32_breakinput { }; struct stm32_pwm { - struct pwm_chip chip; struct mutex lock; /* protect pwm config/enable */ struct clk *clk; struct regmap *regmap; @@ -40,7 +39,7 @@ struct stm32_pwm { static inline struct stm32_pwm *to_stm32_pwm_dev(struct pwm_chip *chip) { - return container_of(chip, struct stm32_pwm, chip); + return pwmchip_get_drvdata(chip); } static u32 active_channels(struct stm32_pwm *dev) @@ -90,11 +89,12 @@ static u32 active_channels(struct stm32_pwm *dev) * - Period = t2 - t0 * - Duty cycle = t1 - t0 */ -static int stm32_pwm_raw_capture(struct stm32_pwm *priv, struct pwm_device *pwm, +static int stm32_pwm_raw_capture(struct pwm_chip *chip, struct pwm_device *pwm, unsigned long tmo_ms, u32 *raw_prd, u32 *raw_dty) { - struct device *parent = priv->chip.dev->parent; + struct stm32_pwm *priv = to_stm32_pwm_dev(chip); + struct device *parent = pwmchip_parent(chip)->parent; enum stm32_timers_dmas dma_id; u32 ccen, ccr; int ret; @@ -170,7 +170,7 @@ static int stm32_pwm_capture(struct pwm_chip *chip, struct pwm_device *pwm, ret = clk_enable(priv->clk); if (ret) { - dev_err(priv->chip.dev, "failed to enable counter clock\n"); + dev_err(pwmchip_parent(chip), "failed to enable counter clock\n"); goto unlock; } @@ -208,7 +208,7 @@ static int stm32_pwm_capture(struct pwm_chip *chip, struct pwm_device *pwm, TIM_CCER_CC12P : TIM_CCER_CC34P, pwm->hwpwm < 2 ? TIM_CCER_CC2P : TIM_CCER_CC4P); - ret = stm32_pwm_raw_capture(priv, pwm, tmo_ms, &raw_prd, &raw_dty); + ret = stm32_pwm_raw_capture(chip, pwm, tmo_ms, &raw_prd, &raw_dty); if (ret) goto stop; @@ -229,7 +229,7 @@ static int stm32_pwm_capture(struct pwm_chip *chip, struct pwm_device *pwm, /* 2nd measure with new scale */ psc /= scale; regmap_write(priv->regmap, TIM_PSC, psc); - ret = stm32_pwm_raw_capture(priv, pwm, tmo_ms, &raw_prd, + ret = stm32_pwm_raw_capture(chip, pwm, tmo_ms, &raw_prd, &raw_dty); if (ret) goto stop; @@ -257,7 +257,7 @@ static int stm32_pwm_capture(struct pwm_chip *chip, struct pwm_device *pwm, FIELD_PREP(TIM_CCMR_IC1PSC, icpsc) | FIELD_PREP(TIM_CCMR_IC2PSC, icpsc)); - ret = stm32_pwm_raw_capture(priv, pwm, tmo_ms, &raw_prd, &raw_dty); + ret = stm32_pwm_raw_capture(chip, pwm, tmo_ms, &raw_prd, &raw_dty); if (ret) goto stop; @@ -605,7 +605,7 @@ static void stm32_pwm_detect_complementary(struct stm32_pwm *priv) priv->have_complementary_output = (ccer != 0); } -static unsigned int stm32_pwm_detect_channels(struct stm32_pwm *priv, +static unsigned int stm32_pwm_detect_channels(struct regmap *regmap, unsigned int *num_enabled) { u32 ccer, ccer_backup; @@ -614,10 +614,10 @@ static unsigned int stm32_pwm_detect_channels(struct stm32_pwm *priv, * If channels enable bits don't exist writing 1 will have no * effect so we can detect and count them. */ - regmap_read(priv->regmap, TIM_CCER, &ccer_backup); - regmap_set_bits(priv->regmap, TIM_CCER, TIM_CCER_CCXE); - regmap_read(priv->regmap, TIM_CCER, &ccer); - regmap_write(priv->regmap, TIM_CCER, ccer_backup); + regmap_read(regmap, TIM_CCER, &ccer_backup); + regmap_set_bits(regmap, TIM_CCER, TIM_CCER_CCXE); + regmap_read(regmap, TIM_CCER, &ccer); + regmap_write(regmap, TIM_CCER, ccer_backup); *num_enabled = hweight32(ccer_backup & TIM_CCER_CCXE); @@ -629,14 +629,18 @@ static int stm32_pwm_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; struct stm32_timers *ddata = dev_get_drvdata(pdev->dev.parent); + struct pwm_chip *chip; struct stm32_pwm *priv; - unsigned int num_enabled; + unsigned int npwm, num_enabled; unsigned int i; int ret; - priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; + npwm = stm32_pwm_detect_channels(ddata->regmap, &num_enabled); + + chip = devm_pwmchip_alloc(dev, npwm, sizeof(*priv)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + priv = to_stm32_pwm_dev(chip); mutex_init(&priv->lock); priv->regmap = ddata->regmap; @@ -652,37 +656,36 @@ static int stm32_pwm_probe(struct platform_device *pdev) stm32_pwm_detect_complementary(priv); - priv->chip.dev = dev; - priv->chip.ops = &stm32pwm_ops; - priv->chip.npwm = stm32_pwm_detect_channels(priv, &num_enabled); + chip->ops = &stm32pwm_ops; /* Initialize clock refcount to number of enabled PWM channels. */ for (i = 0; i < num_enabled; i++) clk_enable(priv->clk); - ret = devm_pwmchip_add(dev, &priv->chip); + ret = devm_pwmchip_add(dev, chip); if (ret < 0) return ret; - platform_set_drvdata(pdev, priv); + platform_set_drvdata(pdev, chip); return 0; } static int stm32_pwm_suspend(struct device *dev) { - struct stm32_pwm *priv = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct stm32_pwm *priv = to_stm32_pwm_dev(chip); unsigned int i; u32 ccer, mask; /* Look for active channels */ ccer = active_channels(priv); - for (i = 0; i < priv->chip.npwm; i++) { + for (i = 0; i < chip->npwm; i++) { mask = TIM_CCER_CC1E << (i * 4); if (ccer & mask) { dev_err(dev, "PWM %u still in use by consumer %s\n", - i, priv->chip.pwms[i].label); + i, chip->pwms[i].label); return -EBUSY; } } @@ -692,7 +695,8 @@ static int stm32_pwm_suspend(struct device *dev) static int stm32_pwm_resume(struct device *dev) { - struct stm32_pwm *priv = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct stm32_pwm *priv = to_stm32_pwm_dev(chip); int ret; ret = pinctrl_pm_select_default_state(dev); diff --git a/drivers/pwm/pwm-stmpe.c b/drivers/pwm/pwm-stmpe.c index 19c0c0f39675..bb91062d5f1d 100644 --- a/drivers/pwm/pwm-stmpe.c +++ b/drivers/pwm/pwm-stmpe.c @@ -27,13 +27,12 @@ struct stmpe_pwm { struct stmpe *stmpe; - struct pwm_chip chip; u8 last_duty; }; static inline struct stmpe_pwm *to_stmpe_pwm(struct pwm_chip *chip) { - return container_of(chip, struct stmpe_pwm, chip); + return pwmchip_get_drvdata(chip); } static int stmpe_24xx_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) @@ -44,7 +43,7 @@ static int stmpe_24xx_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) ret = stmpe_reg_read(stmpe_pwm->stmpe, STMPE24XX_PWMCS); if (ret < 0) { - dev_dbg(chip->dev, "error reading PWM#%u control\n", + dev_dbg(pwmchip_parent(chip), "error reading PWM#%u control\n", pwm->hwpwm); return ret; } @@ -53,7 +52,7 @@ static int stmpe_24xx_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) ret = stmpe_reg_write(stmpe_pwm->stmpe, STMPE24XX_PWMCS, value); if (ret) { - dev_dbg(chip->dev, "error writing PWM#%u control\n", + dev_dbg(pwmchip_parent(chip), "error writing PWM#%u control\n", pwm->hwpwm); return ret; } @@ -70,7 +69,7 @@ static int stmpe_24xx_pwm_disable(struct pwm_chip *chip, ret = stmpe_reg_read(stmpe_pwm->stmpe, STMPE24XX_PWMCS); if (ret < 0) { - dev_dbg(chip->dev, "error reading PWM#%u control\n", + dev_dbg(pwmchip_parent(chip), "error reading PWM#%u control\n", pwm->hwpwm); return ret; } @@ -79,7 +78,7 @@ static int stmpe_24xx_pwm_disable(struct pwm_chip *chip, ret = stmpe_reg_write(stmpe_pwm->stmpe, STMPE24XX_PWMCS, value); if (ret) - dev_dbg(chip->dev, "error writing PWM#%u control\n", + dev_dbg(pwmchip_parent(chip), "error writing PWM#%u control\n", pwm->hwpwm); return ret; } @@ -125,7 +124,7 @@ static int stmpe_24xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, ret = stmpe_set_altfunc(stmpe_pwm->stmpe, BIT(pin), STMPE_BLOCK_PWM); if (ret) { - dev_err(chip->dev, "unable to connect PWM#%u to pin\n", + dev_err(pwmchip_parent(chip), "unable to connect PWM#%u to pin\n", pwm->hwpwm); return ret; } @@ -150,7 +149,7 @@ static int stmpe_24xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, return -ENODEV; } - dev_dbg(chip->dev, "PWM#%u: config duty %d ns, period %d ns\n", + dev_dbg(pwmchip_parent(chip), "PWM#%u: config duty %d ns, period %d ns\n", pwm->hwpwm, duty_ns, period_ns); if (duty_ns == 0) { @@ -216,7 +215,7 @@ static int stmpe_24xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, program[1] = BRANCH; } - dev_dbg(chip->dev, + dev_dbg(pwmchip_parent(chip), "PWM#%u: value = %02x, last_duty = %02x, program=%04x,%04x,%04x\n", pwm->hwpwm, value, last, program[0], program[1], program[2]); @@ -233,7 +232,7 @@ static int stmpe_24xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, ret = stmpe_reg_write(stmpe_pwm->stmpe, offset, value); if (ret) { - dev_dbg(chip->dev, "error writing register %02x: %d\n", + dev_dbg(pwmchip_parent(chip), "error writing register %02x: %d\n", offset, ret); return ret; } @@ -242,7 +241,7 @@ static int stmpe_24xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, ret = stmpe_reg_write(stmpe_pwm->stmpe, offset, value); if (ret) { - dev_dbg(chip->dev, "error writing register %02x: %d\n", + dev_dbg(pwmchip_parent(chip), "error writing register %02x: %d\n", offset, ret); return ret; } @@ -255,7 +254,7 @@ static int stmpe_24xx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, /* Sleep for 200ms so we're sure it will take effect */ msleep(200); - dev_dbg(chip->dev, "programmed PWM#%u, %u bytes\n", pwm->hwpwm, i); + dev_dbg(pwmchip_parent(chip), "programmed PWM#%u, %u bytes\n", pwm->hwpwm, i); return 0; } @@ -292,33 +291,36 @@ static const struct pwm_ops stmpe_24xx_pwm_ops = { static int __init stmpe_pwm_probe(struct platform_device *pdev) { struct stmpe *stmpe = dev_get_drvdata(pdev->dev.parent); + struct pwm_chip *chip; struct stmpe_pwm *stmpe_pwm; int ret; - stmpe_pwm = devm_kzalloc(&pdev->dev, sizeof(*stmpe_pwm), GFP_KERNEL); - if (!stmpe_pwm) - return -ENOMEM; + switch (stmpe->partnum) { + case STMPE2401: + case STMPE2403: + break; + case STMPE1601: + return dev_err_probe(&pdev->dev, -ENODEV, + "STMPE1601 not yet supported\n"); + default: + return dev_err_probe(&pdev->dev, -ENODEV, + "Unknown STMPE PWM\n"); + } + + chip = devm_pwmchip_alloc(&pdev->dev, 3, sizeof(*stmpe_pwm)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + stmpe_pwm = to_stmpe_pwm(chip); stmpe_pwm->stmpe = stmpe; - stmpe_pwm->chip.dev = &pdev->dev; - if (stmpe->partnum == STMPE2401 || stmpe->partnum == STMPE2403) { - stmpe_pwm->chip.ops = &stmpe_24xx_pwm_ops; - stmpe_pwm->chip.npwm = 3; - } else { - if (stmpe->partnum == STMPE1601) - dev_err(&pdev->dev, "STMPE1601 not yet supported\n"); - else - dev_err(&pdev->dev, "Unknown STMPE PWM\n"); - - return -ENODEV; - } + chip->ops = &stmpe_24xx_pwm_ops; ret = stmpe_enable(stmpe, STMPE_BLOCK_PWM); if (ret) return ret; - ret = pwmchip_add(&stmpe_pwm->chip); + ret = pwmchip_add(chip); if (ret) { stmpe_disable(stmpe, STMPE_BLOCK_PWM); return ret; diff --git a/drivers/pwm/pwm-sun4i.c b/drivers/pwm/pwm-sun4i.c index 1a439025540d..5c29590d1821 100644 --- a/drivers/pwm/pwm-sun4i.c +++ b/drivers/pwm/pwm-sun4i.c @@ -81,7 +81,6 @@ struct sun4i_pwm_data { }; struct sun4i_pwm_chip { - struct pwm_chip chip; struct clk *bus_clk; struct clk *clk; struct reset_control *rst; @@ -92,35 +91,35 @@ struct sun4i_pwm_chip { static inline struct sun4i_pwm_chip *to_sun4i_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct sun4i_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } -static inline u32 sun4i_pwm_readl(struct sun4i_pwm_chip *chip, +static inline u32 sun4i_pwm_readl(struct sun4i_pwm_chip *sun4ichip, unsigned long offset) { - return readl(chip->base + offset); + return readl(sun4ichip->base + offset); } -static inline void sun4i_pwm_writel(struct sun4i_pwm_chip *chip, +static inline void sun4i_pwm_writel(struct sun4i_pwm_chip *sun4ichip, u32 val, unsigned long offset) { - writel(val, chip->base + offset); + writel(val, sun4ichip->base + offset); } static int sun4i_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, struct pwm_state *state) { - struct sun4i_pwm_chip *sun4i_pwm = to_sun4i_pwm_chip(chip); + struct sun4i_pwm_chip *sun4ichip = to_sun4i_pwm_chip(chip); u64 clk_rate, tmp; u32 val; unsigned int prescaler; - clk_rate = clk_get_rate(sun4i_pwm->clk); + clk_rate = clk_get_rate(sun4ichip->clk); if (!clk_rate) return -EINVAL; - val = sun4i_pwm_readl(sun4i_pwm, PWM_CTRL_REG); + val = sun4i_pwm_readl(sun4ichip, PWM_CTRL_REG); /* * PWM chapter in H6 manual has a diagram which explains that if bypass @@ -128,7 +127,7 @@ static int sun4i_pwm_get_state(struct pwm_chip *chip, * proved that also enable bit is ignored in this case. */ if ((val & BIT_CH(PWM_BYPASS, pwm->hwpwm)) && - sun4i_pwm->data->has_direct_mod_clk_output) { + sun4ichip->data->has_direct_mod_clk_output) { state->period = DIV_ROUND_UP_ULL(NSEC_PER_SEC, clk_rate); state->duty_cycle = DIV_ROUND_UP_ULL(state->period, 2); state->polarity = PWM_POLARITY_NORMAL; @@ -137,7 +136,7 @@ static int sun4i_pwm_get_state(struct pwm_chip *chip, } if ((PWM_REG_PRESCAL(val, pwm->hwpwm) == PWM_PRESCAL_MASK) && - sun4i_pwm->data->has_prescaler_bypass) + sun4ichip->data->has_prescaler_bypass) prescaler = 1; else prescaler = prescaler_table[PWM_REG_PRESCAL(val, pwm->hwpwm)]; @@ -156,7 +155,7 @@ static int sun4i_pwm_get_state(struct pwm_chip *chip, else state->enabled = false; - val = sun4i_pwm_readl(sun4i_pwm, PWM_CH_PRD(pwm->hwpwm)); + val = sun4i_pwm_readl(sun4ichip, PWM_CH_PRD(pwm->hwpwm)); tmp = (u64)prescaler * NSEC_PER_SEC * PWM_REG_DTY(val); state->duty_cycle = DIV_ROUND_CLOSEST_ULL(tmp, clk_rate); @@ -167,7 +166,7 @@ static int sun4i_pwm_get_state(struct pwm_chip *chip, return 0; } -static int sun4i_pwm_calculate(struct sun4i_pwm_chip *sun4i_pwm, +static int sun4i_pwm_calculate(struct sun4i_pwm_chip *sun4ichip, const struct pwm_state *state, u32 *dty, u32 *prd, unsigned int *prsclr, bool *bypass) @@ -175,9 +174,9 @@ static int sun4i_pwm_calculate(struct sun4i_pwm_chip *sun4i_pwm, u64 clk_rate, div = 0; unsigned int prescaler = 0; - clk_rate = clk_get_rate(sun4i_pwm->clk); + clk_rate = clk_get_rate(sun4ichip->clk); - *bypass = sun4i_pwm->data->has_direct_mod_clk_output && + *bypass = sun4ichip->data->has_direct_mod_clk_output && state->enabled && (state->period * clk_rate >= NSEC_PER_SEC) && (state->period * clk_rate < 2 * NSEC_PER_SEC) && @@ -187,7 +186,7 @@ static int sun4i_pwm_calculate(struct sun4i_pwm_chip *sun4i_pwm, if (*bypass) return 0; - if (sun4i_pwm->data->has_prescaler_bypass) { + if (sun4ichip->data->has_prescaler_bypass) { /* First, test without any prescaler when available */ prescaler = PWM_PRESCAL_MASK; /* @@ -233,7 +232,7 @@ static int sun4i_pwm_calculate(struct sun4i_pwm_chip *sun4i_pwm, static int sun4i_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { - struct sun4i_pwm_chip *sun4i_pwm = to_sun4i_pwm_chip(chip); + struct sun4i_pwm_chip *sun4ichip = to_sun4i_pwm_chip(chip); struct pwm_state cstate; u32 ctrl, duty = 0, period = 0, val; int ret; @@ -243,31 +242,31 @@ static int sun4i_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, pwm_get_state(pwm, &cstate); if (!cstate.enabled) { - ret = clk_prepare_enable(sun4i_pwm->clk); + ret = clk_prepare_enable(sun4ichip->clk); if (ret) { - dev_err(chip->dev, "failed to enable PWM clock\n"); + dev_err(pwmchip_parent(chip), "failed to enable PWM clock\n"); return ret; } } - ret = sun4i_pwm_calculate(sun4i_pwm, state, &duty, &period, &prescaler, + ret = sun4i_pwm_calculate(sun4ichip, state, &duty, &period, &prescaler, &bypass); if (ret) { - dev_err(chip->dev, "period exceeds the maximum value\n"); + dev_err(pwmchip_parent(chip), "period exceeds the maximum value\n"); if (!cstate.enabled) - clk_disable_unprepare(sun4i_pwm->clk); + clk_disable_unprepare(sun4ichip->clk); return ret; } - spin_lock(&sun4i_pwm->ctrl_lock); - ctrl = sun4i_pwm_readl(sun4i_pwm, PWM_CTRL_REG); + spin_lock(&sun4ichip->ctrl_lock); + ctrl = sun4i_pwm_readl(sun4ichip, PWM_CTRL_REG); - if (sun4i_pwm->data->has_direct_mod_clk_output) { + if (sun4ichip->data->has_direct_mod_clk_output) { if (bypass) { ctrl |= BIT_CH(PWM_BYPASS, pwm->hwpwm); /* We can skip other parameter */ - sun4i_pwm_writel(sun4i_pwm, ctrl, PWM_CTRL_REG); - spin_unlock(&sun4i_pwm->ctrl_lock); + sun4i_pwm_writel(sun4ichip, ctrl, PWM_CTRL_REG); + spin_unlock(&sun4ichip->ctrl_lock); return 0; } @@ -277,14 +276,14 @@ static int sun4i_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (PWM_REG_PRESCAL(ctrl, pwm->hwpwm) != prescaler) { /* Prescaler changed, the clock has to be gated */ ctrl &= ~BIT_CH(PWM_CLK_GATING, pwm->hwpwm); - sun4i_pwm_writel(sun4i_pwm, ctrl, PWM_CTRL_REG); + sun4i_pwm_writel(sun4ichip, ctrl, PWM_CTRL_REG); ctrl &= ~BIT_CH(PWM_PRESCAL_MASK, pwm->hwpwm); ctrl |= BIT_CH(prescaler, pwm->hwpwm); } val = (duty & PWM_DTY_MASK) | PWM_PRD(period); - sun4i_pwm_writel(sun4i_pwm, val, PWM_CH_PRD(pwm->hwpwm)); + sun4i_pwm_writel(sun4ichip, val, PWM_CH_PRD(pwm->hwpwm)); if (state->polarity != PWM_POLARITY_NORMAL) ctrl &= ~BIT_CH(PWM_ACT_STATE, pwm->hwpwm); @@ -296,9 +295,9 @@ static int sun4i_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (state->enabled) ctrl |= BIT_CH(PWM_EN, pwm->hwpwm); - sun4i_pwm_writel(sun4i_pwm, ctrl, PWM_CTRL_REG); + sun4i_pwm_writel(sun4ichip, ctrl, PWM_CTRL_REG); - spin_unlock(&sun4i_pwm->ctrl_lock); + spin_unlock(&sun4ichip->ctrl_lock); if (state->enabled) return 0; @@ -310,14 +309,14 @@ static int sun4i_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, else usleep_range(delay_us, delay_us * 2); - spin_lock(&sun4i_pwm->ctrl_lock); - ctrl = sun4i_pwm_readl(sun4i_pwm, PWM_CTRL_REG); + spin_lock(&sun4ichip->ctrl_lock); + ctrl = sun4i_pwm_readl(sun4ichip, PWM_CTRL_REG); ctrl &= ~BIT_CH(PWM_CLK_GATING, pwm->hwpwm); ctrl &= ~BIT_CH(PWM_EN, pwm->hwpwm); - sun4i_pwm_writel(sun4i_pwm, ctrl, PWM_CTRL_REG); - spin_unlock(&sun4i_pwm->ctrl_lock); + sun4i_pwm_writel(sun4ichip, ctrl, PWM_CTRL_REG); + spin_unlock(&sun4ichip->ctrl_lock); - clk_disable_unprepare(sun4i_pwm->clk); + clk_disable_unprepare(sun4ichip->clk); return 0; } @@ -384,17 +383,21 @@ MODULE_DEVICE_TABLE(of, sun4i_pwm_dt_ids); static int sun4i_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; + const struct sun4i_pwm_data *data; struct sun4i_pwm_chip *sun4ichip; int ret; - sun4ichip = devm_kzalloc(&pdev->dev, sizeof(*sun4ichip), GFP_KERNEL); - if (!sun4ichip) - return -ENOMEM; - - sun4ichip->data = of_device_get_match_data(&pdev->dev); - if (!sun4ichip->data) + data = of_device_get_match_data(&pdev->dev); + if (!data) return -ENODEV; + chip = devm_pwmchip_alloc(&pdev->dev, data->npwm, sizeof(*sun4ichip)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + sun4ichip = to_sun4i_pwm_chip(chip); + + sun4ichip->data = data; sun4ichip->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(sun4ichip->base)) return PTR_ERR(sun4ichip->base); @@ -451,19 +454,17 @@ static int sun4i_pwm_probe(struct platform_device *pdev) goto err_bus; } - sun4ichip->chip.dev = &pdev->dev; - sun4ichip->chip.ops = &sun4i_pwm_ops; - sun4ichip->chip.npwm = sun4ichip->data->npwm; + chip->ops = &sun4i_pwm_ops; spin_lock_init(&sun4ichip->ctrl_lock); - ret = pwmchip_add(&sun4ichip->chip); + ret = pwmchip_add(chip); if (ret < 0) { dev_err(&pdev->dev, "failed to add PWM chip: %d\n", ret); goto err_pwm_add; } - platform_set_drvdata(pdev, sun4ichip); + platform_set_drvdata(pdev, chip); return 0; @@ -477,9 +478,10 @@ err_bus: static void sun4i_pwm_remove(struct platform_device *pdev) { - struct sun4i_pwm_chip *sun4ichip = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct sun4i_pwm_chip *sun4ichip = to_sun4i_pwm_chip(chip); - pwmchip_remove(&sun4ichip->chip); + pwmchip_remove(chip); clk_disable_unprepare(sun4ichip->bus_clk); reset_control_assert(sun4ichip->rst); diff --git a/drivers/pwm/pwm-sunplus.c b/drivers/pwm/pwm-sunplus.c index 773e2f80526e..b342b843247b 100644 --- a/drivers/pwm/pwm-sunplus.c +++ b/drivers/pwm/pwm-sunplus.c @@ -43,14 +43,13 @@ #define SP7021_PWM_NUM 4 struct sunplus_pwm { - struct pwm_chip chip; void __iomem *base; struct clk *clk; }; static inline struct sunplus_pwm *to_sunplus_pwm(struct pwm_chip *chip) { - return container_of(chip, struct sunplus_pwm, chip); + return pwmchip_get_drvdata(chip); } static int sunplus_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, @@ -175,12 +174,14 @@ static void sunplus_pwm_clk_release(void *data) static int sunplus_pwm_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + struct pwm_chip *chip; struct sunplus_pwm *priv; int ret; - priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; + chip = devm_pwmchip_alloc(dev, SP7021_PWM_NUM, sizeof(*priv)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + priv = to_sunplus_pwm(chip); priv->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(priv->base)) @@ -203,11 +204,9 @@ static int sunplus_pwm_probe(struct platform_device *pdev) return ret; } - priv->chip.dev = dev; - priv->chip.ops = &sunplus_pwm_ops; - priv->chip.npwm = SP7021_PWM_NUM; + chip->ops = &sunplus_pwm_ops; - ret = devm_pwmchip_add(dev, &priv->chip); + ret = devm_pwmchip_add(dev, chip); if (ret < 0) return dev_err_probe(dev, ret, "Cannot register sunplus PWM\n"); diff --git a/drivers/pwm/pwm-tegra.c b/drivers/pwm/pwm-tegra.c index 82ee2f0754f9..a3d69976148f 100644 --- a/drivers/pwm/pwm-tegra.c +++ b/drivers/pwm/pwm-tegra.c @@ -65,9 +65,6 @@ struct tegra_pwm_soc { }; struct tegra_pwm_chip { - struct pwm_chip chip; - struct device *dev; - struct clk *clk; struct reset_control*rst; @@ -81,7 +78,7 @@ struct tegra_pwm_chip { static inline struct tegra_pwm_chip *to_tegra_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct tegra_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static inline u32 pwm_readl(struct tegra_pwm_chip *pc, unsigned int offset) @@ -158,7 +155,7 @@ static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, */ required_clk_rate *= 2; - err = dev_pm_opp_set_rate(pc->dev, required_clk_rate); + err = dev_pm_opp_set_rate(pwmchip_parent(chip), required_clk_rate); if (err < 0) return -EINVAL; @@ -194,7 +191,7 @@ static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, * before writing the register. Otherwise, keep it enabled. */ if (!pwm_is_enabled(pwm)) { - err = pm_runtime_resume_and_get(pc->dev); + err = pm_runtime_resume_and_get(pwmchip_parent(chip)); if (err) return err; } else @@ -206,7 +203,7 @@ static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, * If the PWM is not enabled, turn the clock off again to save power. */ if (!pwm_is_enabled(pwm)) - pm_runtime_put(pc->dev); + pm_runtime_put(pwmchip_parent(chip)); return 0; } @@ -217,7 +214,7 @@ static int tegra_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) int rc = 0; u32 val; - rc = pm_runtime_resume_and_get(pc->dev); + rc = pm_runtime_resume_and_get(pwmchip_parent(chip)); if (rc) return rc; @@ -237,7 +234,7 @@ static void tegra_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) val &= ~PWM_ENABLE; pwm_writel(pc, pwm->hwpwm, val); - pm_runtime_put_sync(pc->dev); + pm_runtime_put_sync(pwmchip_parent(chip)); } static int tegra_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, @@ -272,21 +269,25 @@ static const struct pwm_ops tegra_pwm_ops = { static int tegra_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct tegra_pwm_chip *pc; + const struct tegra_pwm_soc *soc; int ret; - pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL); - if (!pc) - return -ENOMEM; + soc = of_device_get_match_data(&pdev->dev); - pc->soc = of_device_get_match_data(&pdev->dev); - pc->dev = &pdev->dev; + chip = devm_pwmchip_alloc(&pdev->dev, soc->num_channels, sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_tegra_pwm_chip(chip); + + pc->soc = soc; pc->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->regs)) return PTR_ERR(pc->regs); - platform_set_drvdata(pdev, pc); + platform_set_drvdata(pdev, chip); pc->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(pc->clk)) @@ -302,7 +303,7 @@ static int tegra_pwm_probe(struct platform_device *pdev) return ret; /* Set maximum frequency of the IP */ - ret = dev_pm_opp_set_rate(pc->dev, pc->soc->max_frequency); + ret = dev_pm_opp_set_rate(&pdev->dev, pc->soc->max_frequency); if (ret < 0) { dev_err(&pdev->dev, "Failed to set max frequency: %d\n", ret); goto put_pm; @@ -328,11 +329,9 @@ static int tegra_pwm_probe(struct platform_device *pdev) reset_control_deassert(pc->rst); - pc->chip.dev = &pdev->dev; - pc->chip.ops = &tegra_pwm_ops; - pc->chip.npwm = pc->soc->num_channels; + chip->ops = &tegra_pwm_ops; - ret = pwmchip_add(&pc->chip); + ret = pwmchip_add(chip); if (ret < 0) { dev_err(&pdev->dev, "pwmchip_add() failed: %d\n", ret); reset_control_assert(pc->rst); @@ -350,9 +349,10 @@ put_pm: static void tegra_pwm_remove(struct platform_device *pdev) { - struct tegra_pwm_chip *pc = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct tegra_pwm_chip *pc = to_tegra_pwm_chip(chip); - pwmchip_remove(&pc->chip); + pwmchip_remove(chip); reset_control_assert(pc->rst); @@ -361,7 +361,8 @@ static void tegra_pwm_remove(struct platform_device *pdev) static int __maybe_unused tegra_pwm_runtime_suspend(struct device *dev) { - struct tegra_pwm_chip *pc = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct tegra_pwm_chip *pc = to_tegra_pwm_chip(chip); int err; clk_disable_unprepare(pc->clk); @@ -377,7 +378,8 @@ static int __maybe_unused tegra_pwm_runtime_suspend(struct device *dev) static int __maybe_unused tegra_pwm_runtime_resume(struct device *dev) { - struct tegra_pwm_chip *pc = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); + struct tegra_pwm_chip *pc = to_tegra_pwm_chip(chip); int err; err = pinctrl_pm_select_default_state(dev); diff --git a/drivers/pwm/pwm-tiecap.c b/drivers/pwm/pwm-tiecap.c index d974f4414ac9..d6c2b1b1387e 100644 --- a/drivers/pwm/pwm-tiecap.c +++ b/drivers/pwm/pwm-tiecap.c @@ -32,7 +32,6 @@ struct ecap_context { }; struct ecap_pwm_chip { - struct pwm_chip chip; unsigned int clk_rate; void __iomem *mmio_base; struct ecap_context ctx; @@ -40,7 +39,7 @@ struct ecap_pwm_chip { static inline struct ecap_pwm_chip *to_ecap_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct ecap_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } /* @@ -70,7 +69,7 @@ static int ecap_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, duty_cycles = (u32)c; } - pm_runtime_get_sync(pc->chip.dev); + pm_runtime_get_sync(pwmchip_parent(chip)); value = readw(pc->mmio_base + ECCTL2); @@ -100,7 +99,7 @@ static int ecap_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, writew(value, pc->mmio_base + ECCTL2); } - pm_runtime_put_sync(pc->chip.dev); + pm_runtime_put_sync(pwmchip_parent(chip)); return 0; } @@ -111,7 +110,7 @@ static int ecap_pwm_set_polarity(struct pwm_chip *chip, struct pwm_device *pwm, struct ecap_pwm_chip *pc = to_ecap_pwm_chip(chip); u16 value; - pm_runtime_get_sync(pc->chip.dev); + pm_runtime_get_sync(pwmchip_parent(chip)); value = readw(pc->mmio_base + ECCTL2); @@ -124,7 +123,7 @@ static int ecap_pwm_set_polarity(struct pwm_chip *chip, struct pwm_device *pwm, writew(value, pc->mmio_base + ECCTL2); - pm_runtime_put_sync(pc->chip.dev); + pm_runtime_put_sync(pwmchip_parent(chip)); return 0; } @@ -135,7 +134,7 @@ static int ecap_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) u16 value; /* Leave clock enabled on enabling PWM */ - pm_runtime_get_sync(pc->chip.dev); + pm_runtime_get_sync(pwmchip_parent(chip)); /* * Enable 'Free run Time stamp counter mode' to start counter @@ -162,7 +161,7 @@ static void ecap_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) writew(value, pc->mmio_base + ECCTL2); /* Disable clock on PWM disable */ - pm_runtime_put_sync(pc->chip.dev); + pm_runtime_put_sync(pwmchip_parent(chip)); } static int ecap_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, @@ -218,12 +217,14 @@ static int ecap_pwm_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct ecap_pwm_chip *pc; + struct pwm_chip *chip; struct clk *clk; int ret; - pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL); - if (!pc) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_ecap_pwm_chip(chip); clk = devm_clk_get(&pdev->dev, "fck"); if (IS_ERR(clk)) { @@ -244,21 +245,19 @@ static int ecap_pwm_probe(struct platform_device *pdev) return -EINVAL; } - pc->chip.dev = &pdev->dev; - pc->chip.ops = &ecap_pwm_ops; - pc->chip.npwm = 1; + chip->ops = &ecap_pwm_ops; pc->mmio_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->mmio_base)) return PTR_ERR(pc->mmio_base); - ret = devm_pwmchip_add(&pdev->dev, &pc->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) { dev_err(&pdev->dev, "pwmchip_add() failed: %d\n", ret); return ret; } - platform_set_drvdata(pdev, pc); + platform_set_drvdata(pdev, chip); pm_runtime_enable(&pdev->dev); return 0; @@ -269,17 +268,21 @@ static void ecap_pwm_remove(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); } -static void ecap_pwm_save_context(struct ecap_pwm_chip *pc) +static void ecap_pwm_save_context(struct pwm_chip *chip) { - pm_runtime_get_sync(pc->chip.dev); + struct ecap_pwm_chip *pc = to_ecap_pwm_chip(chip); + + pm_runtime_get_sync(pwmchip_parent(chip)); pc->ctx.ecctl2 = readw(pc->mmio_base + ECCTL2); pc->ctx.cap4 = readl(pc->mmio_base + CAP4); pc->ctx.cap3 = readl(pc->mmio_base + CAP3); - pm_runtime_put_sync(pc->chip.dev); + pm_runtime_put_sync(pwmchip_parent(chip)); } -static void ecap_pwm_restore_context(struct ecap_pwm_chip *pc) +static void ecap_pwm_restore_context(struct pwm_chip *chip) { + struct ecap_pwm_chip *pc = to_ecap_pwm_chip(chip); + writel(pc->ctx.cap3, pc->mmio_base + CAP3); writel(pc->ctx.cap4, pc->mmio_base + CAP4); writew(pc->ctx.ecctl2, pc->mmio_base + ECCTL2); @@ -287,10 +290,10 @@ static void ecap_pwm_restore_context(struct ecap_pwm_chip *pc) static int ecap_pwm_suspend(struct device *dev) { - struct ecap_pwm_chip *pc = dev_get_drvdata(dev); - struct pwm_device *pwm = pc->chip.pwms; + struct pwm_chip *chip = dev_get_drvdata(dev); + struct pwm_device *pwm = chip->pwms; - ecap_pwm_save_context(pc); + ecap_pwm_save_context(chip); /* Disable explicitly if PWM is running */ if (pwm_is_enabled(pwm)) @@ -301,14 +304,14 @@ static int ecap_pwm_suspend(struct device *dev) static int ecap_pwm_resume(struct device *dev) { - struct ecap_pwm_chip *pc = dev_get_drvdata(dev); - struct pwm_device *pwm = pc->chip.pwms; + struct pwm_chip *chip = dev_get_drvdata(dev); + struct pwm_device *pwm = chip->pwms; /* Enable explicitly if PWM was running */ if (pwm_is_enabled(pwm)) pm_runtime_get_sync(dev); - ecap_pwm_restore_context(pc); + ecap_pwm_restore_context(chip); return 0; } diff --git a/drivers/pwm/pwm-tiehrpwm.c b/drivers/pwm/pwm-tiehrpwm.c index af231fa74fa9..e5104725d9b7 100644 --- a/drivers/pwm/pwm-tiehrpwm.c +++ b/drivers/pwm/pwm-tiehrpwm.c @@ -105,7 +105,6 @@ struct ehrpwm_context { }; struct ehrpwm_pwm_chip { - struct pwm_chip chip; unsigned long clk_rate; void __iomem *mmio_base; unsigned long period_cycles[NUM_PWM_CHANNEL]; @@ -116,7 +115,7 @@ struct ehrpwm_pwm_chip { static inline struct ehrpwm_pwm_chip *to_ehrpwm_pwm_chip(struct pwm_chip *chip) { - return container_of(chip, struct ehrpwm_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static inline u16 ehrpwm_read(void __iomem *base, unsigned int offset) @@ -256,7 +255,7 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if (i == pwm->hwpwm) continue; - dev_err(chip->dev, + dev_err(pwmchip_parent(chip), "period value conflicts with channel %u\n", i); return -EINVAL; @@ -268,11 +267,11 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, /* Configure clock prescaler to support Low frequency PWM wave */ if (set_prescale_div(period_cycles/PERIOD_MAX, &ps_divval, &tb_divval)) { - dev_err(chip->dev, "Unsupported values\n"); + dev_err(pwmchip_parent(chip), "Unsupported values\n"); return -EINVAL; } - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); /* Update clock prescaler values */ ehrpwm_modify(pc->mmio_base, TBCTL, TBCTL_CLKDIV_MASK, tb_divval); @@ -299,7 +298,7 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, ehrpwm_write(pc->mmio_base, cmp_reg, duty_cycles); - pm_runtime_put_sync(chip->dev); + pm_runtime_put_sync(pwmchip_parent(chip)); return 0; } @@ -323,7 +322,7 @@ static int ehrpwm_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) int ret; /* Leave clock enabled on enabling PWM */ - pm_runtime_get_sync(chip->dev); + pm_runtime_get_sync(pwmchip_parent(chip)); /* Disabling Action Qualifier on PWM output */ if (pwm->hwpwm) { @@ -346,8 +345,8 @@ static int ehrpwm_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) /* Enable TBCLK */ ret = clk_enable(pc->tbclk); if (ret) { - dev_err(chip->dev, "Failed to enable TBCLK for %s: %d\n", - dev_name(pc->chip.dev), ret); + dev_err(pwmchip_parent(chip), "Failed to enable TBCLK for %s: %d\n", + dev_name(pwmchip_parent(chip)), ret); return ret; } @@ -385,7 +384,7 @@ static void ehrpwm_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) clk_disable(pc->tbclk); /* Disable clock on PWM disable */ - pm_runtime_put_sync(chip->dev); + pm_runtime_put_sync(pwmchip_parent(chip)); } static void ehrpwm_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) @@ -393,8 +392,8 @@ static void ehrpwm_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) struct ehrpwm_pwm_chip *pc = to_ehrpwm_pwm_chip(chip); if (pwm_is_enabled(pwm)) { - dev_warn(chip->dev, "Removing PWM device without disabling\n"); - pm_runtime_put_sync(chip->dev); + dev_warn(pwmchip_parent(chip), "Removing PWM device without disabling\n"); + pm_runtime_put_sync(pwmchip_parent(chip)); } /* set period value to zero on free */ @@ -450,12 +449,14 @@ static int ehrpwm_pwm_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct ehrpwm_pwm_chip *pc; + struct pwm_chip *chip; struct clk *clk; int ret; - pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL); - if (!pc) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, NUM_PWM_CHANNEL, sizeof(*pc)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + pc = to_ehrpwm_pwm_chip(chip); clk = devm_clk_get(&pdev->dev, "fck"); if (IS_ERR(clk)) { @@ -474,9 +475,7 @@ static int ehrpwm_pwm_probe(struct platform_device *pdev) return -EINVAL; } - pc->chip.dev = &pdev->dev; - pc->chip.ops = &ehrpwm_pwm_ops; - pc->chip.npwm = NUM_PWM_CHANNEL; + chip->ops = &ehrpwm_pwm_ops; pc->mmio_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pc->mmio_base)) @@ -493,13 +492,13 @@ static int ehrpwm_pwm_probe(struct platform_device *pdev) return ret; } - ret = pwmchip_add(&pc->chip); + ret = pwmchip_add(chip); if (ret < 0) { dev_err(&pdev->dev, "pwmchip_add() failed: %d\n", ret); goto err_clk_unprepare; } - platform_set_drvdata(pdev, pc); + platform_set_drvdata(pdev, chip); pm_runtime_enable(&pdev->dev); return 0; @@ -512,18 +511,21 @@ err_clk_unprepare: static void ehrpwm_pwm_remove(struct platform_device *pdev) { - struct ehrpwm_pwm_chip *pc = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct ehrpwm_pwm_chip *pc = to_ehrpwm_pwm_chip(chip); - pwmchip_remove(&pc->chip); + pwmchip_remove(chip); clk_unprepare(pc->tbclk); pm_runtime_disable(&pdev->dev); } -static void ehrpwm_pwm_save_context(struct ehrpwm_pwm_chip *pc) +static void ehrpwm_pwm_save_context(struct pwm_chip *chip) { - pm_runtime_get_sync(pc->chip.dev); + struct ehrpwm_pwm_chip *pc = to_ehrpwm_pwm_chip(chip); + + pm_runtime_get_sync(pwmchip_parent(chip)); pc->ctx.tbctl = ehrpwm_read(pc->mmio_base, TBCTL); pc->ctx.tbprd = ehrpwm_read(pc->mmio_base, TBPRD); @@ -534,11 +536,13 @@ static void ehrpwm_pwm_save_context(struct ehrpwm_pwm_chip *pc) pc->ctx.aqsfrc = ehrpwm_read(pc->mmio_base, AQSFRC); pc->ctx.aqcsfrc = ehrpwm_read(pc->mmio_base, AQCSFRC); - pm_runtime_put_sync(pc->chip.dev); + pm_runtime_put_sync(pwmchip_parent(chip)); } -static void ehrpwm_pwm_restore_context(struct ehrpwm_pwm_chip *pc) +static void ehrpwm_pwm_restore_context(struct pwm_chip *chip) { + struct ehrpwm_pwm_chip *pc = to_ehrpwm_pwm_chip(chip); + ehrpwm_write(pc->mmio_base, TBPRD, pc->ctx.tbprd); ehrpwm_write(pc->mmio_base, CMPA, pc->ctx.cmpa); ehrpwm_write(pc->mmio_base, CMPB, pc->ctx.cmpb); @@ -551,13 +555,13 @@ static void ehrpwm_pwm_restore_context(struct ehrpwm_pwm_chip *pc) static int ehrpwm_pwm_suspend(struct device *dev) { - struct ehrpwm_pwm_chip *pc = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); unsigned int i; - ehrpwm_pwm_save_context(pc); + ehrpwm_pwm_save_context(chip); - for (i = 0; i < pc->chip.npwm; i++) { - struct pwm_device *pwm = &pc->chip.pwms[i]; + for (i = 0; i < chip->npwm; i++) { + struct pwm_device *pwm = &chip->pwms[i]; if (!pwm_is_enabled(pwm)) continue; @@ -571,11 +575,11 @@ static int ehrpwm_pwm_suspend(struct device *dev) static int ehrpwm_pwm_resume(struct device *dev) { - struct ehrpwm_pwm_chip *pc = dev_get_drvdata(dev); + struct pwm_chip *chip = dev_get_drvdata(dev); unsigned int i; - for (i = 0; i < pc->chip.npwm; i++) { - struct pwm_device *pwm = &pc->chip.pwms[i]; + for (i = 0; i < chip->npwm; i++) { + struct pwm_device *pwm = &chip->pwms[i]; if (!pwm_is_enabled(pwm)) continue; @@ -584,7 +588,7 @@ static int ehrpwm_pwm_resume(struct device *dev) pm_runtime_get_sync(dev); } - ehrpwm_pwm_restore_context(pc); + ehrpwm_pwm_restore_context(chip); return 0; } diff --git a/drivers/pwm/pwm-twl-led.c b/drivers/pwm/pwm-twl-led.c index c670ccb81653..4b10a8dab312 100644 --- a/drivers/pwm/pwm-twl-led.c +++ b/drivers/pwm/pwm-twl-led.c @@ -62,13 +62,12 @@ #define TWL6040_LED_MODE_MASK 0x03 struct twl_pwmled_chip { - struct pwm_chip chip; struct mutex mutex; }; static inline struct twl_pwmled_chip *to_twl(struct pwm_chip *chip) { - return container_of(chip, struct twl_pwmled_chip, chip); + return pwmchip_get_drvdata(chip); } static int twl4030_pwmled_config(struct pwm_chip *chip, struct pwm_device *pwm, @@ -100,7 +99,7 @@ static int twl4030_pwmled_config(struct pwm_chip *chip, struct pwm_device *pwm, ret = twl_i2c_write(TWL4030_MODULE_LED, pwm_config, base, 2); if (ret < 0) - dev_err(chip->dev, "%s: Failed to configure PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to configure PWM\n", pwm->label); return ret; } @@ -114,7 +113,7 @@ static int twl4030_pwmled_enable(struct pwm_chip *chip, struct pwm_device *pwm) mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL4030_MODULE_LED, &val, TWL4030_LEDEN_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read LEDEN\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to read LEDEN\n", pwm->label); goto out; } @@ -122,7 +121,7 @@ static int twl4030_pwmled_enable(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL4030_MODULE_LED, val, TWL4030_LEDEN_REG); if (ret < 0) - dev_err(chip->dev, "%s: Failed to enable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to enable PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -139,7 +138,7 @@ static void twl4030_pwmled_disable(struct pwm_chip *chip, mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL4030_MODULE_LED, &val, TWL4030_LEDEN_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read LEDEN\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to read LEDEN\n", pwm->label); goto out; } @@ -147,7 +146,7 @@ static void twl4030_pwmled_disable(struct pwm_chip *chip, ret = twl_i2c_write_u8(TWL4030_MODULE_LED, val, TWL4030_LEDEN_REG); if (ret < 0) - dev_err(chip->dev, "%s: Failed to disable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to disable PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -203,7 +202,7 @@ static int twl6030_pwmled_config(struct pwm_chip *chip, struct pwm_device *pwm, ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, on_time, TWL6030_LED_PWM_CTRL1); if (ret < 0) - dev_err(chip->dev, "%s: Failed to configure PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to configure PWM\n", pwm->label); return ret; } @@ -217,7 +216,7 @@ static int twl6030_pwmled_enable(struct pwm_chip *chip, struct pwm_device *pwm) mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL6030_MODULE_ID1, &val, TWL6030_LED_PWM_CTRL2); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read PWM_CTRL2\n", + dev_err(pwmchip_parent(chip), "%s: Failed to read PWM_CTRL2\n", pwm->label); goto out; } @@ -227,7 +226,7 @@ static int twl6030_pwmled_enable(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, TWL6030_LED_PWM_CTRL2); if (ret < 0) - dev_err(chip->dev, "%s: Failed to enable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to enable PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -244,7 +243,7 @@ static void twl6030_pwmled_disable(struct pwm_chip *chip, mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL6030_MODULE_ID1, &val, TWL6030_LED_PWM_CTRL2); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read PWM_CTRL2\n", + dev_err(pwmchip_parent(chip), "%s: Failed to read PWM_CTRL2\n", pwm->label); goto out; } @@ -254,7 +253,7 @@ static void twl6030_pwmled_disable(struct pwm_chip *chip, ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, TWL6030_LED_PWM_CTRL2); if (ret < 0) - dev_err(chip->dev, "%s: Failed to disable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to disable PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -295,7 +294,7 @@ static int twl6030_pwmled_request(struct pwm_chip *chip, struct pwm_device *pwm) mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL6030_MODULE_ID1, &val, TWL6030_LED_PWM_CTRL2); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read PWM_CTRL2\n", + dev_err(pwmchip_parent(chip), "%s: Failed to read PWM_CTRL2\n", pwm->label); goto out; } @@ -305,7 +304,7 @@ static int twl6030_pwmled_request(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, TWL6030_LED_PWM_CTRL2); if (ret < 0) - dev_err(chip->dev, "%s: Failed to request PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to request PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -321,7 +320,7 @@ static void twl6030_pwmled_free(struct pwm_chip *chip, struct pwm_device *pwm) mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL6030_MODULE_ID1, &val, TWL6030_LED_PWM_CTRL2); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read PWM_CTRL2\n", + dev_err(pwmchip_parent(chip), "%s: Failed to read PWM_CTRL2\n", pwm->label); goto out; } @@ -331,7 +330,7 @@ static void twl6030_pwmled_free(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, TWL6030_LED_PWM_CTRL2); if (ret < 0) - dev_err(chip->dev, "%s: Failed to free PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to free PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -345,25 +344,29 @@ static const struct pwm_ops twl6030_pwmled_ops = { static int twl_pwmled_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct twl_pwmled_chip *twl; - - twl = devm_kzalloc(&pdev->dev, sizeof(*twl), GFP_KERNEL); - if (!twl) - return -ENOMEM; + unsigned int npwm; + const struct pwm_ops *ops; if (twl_class_is_4030()) { - twl->chip.ops = &twl4030_pwmled_ops; - twl->chip.npwm = 2; + ops = &twl4030_pwmled_ops; + npwm = 2; } else { - twl->chip.ops = &twl6030_pwmled_ops; - twl->chip.npwm = 1; + ops = &twl6030_pwmled_ops; + npwm = 1; } - twl->chip.dev = &pdev->dev; + chip = devm_pwmchip_alloc(&pdev->dev, npwm, sizeof(*twl)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + twl = to_twl(chip); + + chip->ops = ops; mutex_init(&twl->mutex); - return devm_pwmchip_add(&pdev->dev, &twl->chip); + return devm_pwmchip_add(&pdev->dev, chip); } #ifdef CONFIG_OF diff --git a/drivers/pwm/pwm-twl.c b/drivers/pwm/pwm-twl.c index 68e02c9a6bf9..8f981ffff4b4 100644 --- a/drivers/pwm/pwm-twl.c +++ b/drivers/pwm/pwm-twl.c @@ -46,7 +46,6 @@ #define TWL6030_PWM_TOGGLE(pwm, x) ((x) << (pwm * 3)) struct twl_pwm_chip { - struct pwm_chip chip; struct mutex mutex; u8 twl6030_toggle3; u8 twl4030_pwm_mux; @@ -54,7 +53,7 @@ struct twl_pwm_chip { static inline struct twl_pwm_chip *to_twl(struct pwm_chip *chip) { - return container_of(chip, struct twl_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static int twl_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, @@ -86,7 +85,7 @@ static int twl_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, ret = twl_i2c_write(TWL_MODULE_PWM, pwm_config, base, 2); if (ret < 0) - dev_err(chip->dev, "%s: Failed to configure PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to configure PWM\n", pwm->label); return ret; } @@ -100,7 +99,7 @@ static int twl4030_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL4030_MODULE_INTBR, &val, TWL4030_GPBR1_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read GPBR1\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to read GPBR1\n", pwm->label); goto out; } @@ -108,13 +107,13 @@ static int twl4030_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL4030_MODULE_INTBR, val, TWL4030_GPBR1_REG); if (ret < 0) - dev_err(chip->dev, "%s: Failed to enable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to enable PWM\n", pwm->label); val |= TWL4030_PWM_TOGGLE(pwm->hwpwm, TWL4030_PWMX_ENABLE); ret = twl_i2c_write_u8(TWL4030_MODULE_INTBR, val, TWL4030_GPBR1_REG); if (ret < 0) - dev_err(chip->dev, "%s: Failed to enable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to enable PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -130,7 +129,7 @@ static void twl4030_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL4030_MODULE_INTBR, &val, TWL4030_GPBR1_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read GPBR1\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to read GPBR1\n", pwm->label); goto out; } @@ -138,13 +137,13 @@ static void twl4030_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL4030_MODULE_INTBR, val, TWL4030_GPBR1_REG); if (ret < 0) - dev_err(chip->dev, "%s: Failed to disable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to disable PWM\n", pwm->label); val &= ~TWL4030_PWM_TOGGLE(pwm->hwpwm, TWL4030_PWMXCLK_ENABLE); ret = twl_i2c_write_u8(TWL4030_MODULE_INTBR, val, TWL4030_GPBR1_REG); if (ret < 0) - dev_err(chip->dev, "%s: Failed to disable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to disable PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -167,7 +166,7 @@ static int twl4030_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL4030_MODULE_INTBR, &val, TWL4030_PMBR1_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read PMBR1\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to read PMBR1\n", pwm->label); goto out; } @@ -181,7 +180,7 @@ static int twl4030_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL4030_MODULE_INTBR, val, TWL4030_PMBR1_REG); if (ret < 0) - dev_err(chip->dev, "%s: Failed to request PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to request PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -202,7 +201,7 @@ static void twl4030_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) mutex_lock(&twl->mutex); ret = twl_i2c_read_u8(TWL4030_MODULE_INTBR, &val, TWL4030_PMBR1_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to read PMBR1\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to read PMBR1\n", pwm->label); goto out; } @@ -212,7 +211,7 @@ static void twl4030_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL4030_MODULE_INTBR, val, TWL4030_PMBR1_REG); if (ret < 0) - dev_err(chip->dev, "%s: Failed to free PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to free PWM\n", pwm->label); out: mutex_unlock(&twl->mutex); @@ -231,7 +230,7 @@ static int twl6030_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, TWL6030_TOGGLE3_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to enable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to enable PWM\n", pwm->label); goto out; } @@ -254,7 +253,7 @@ static void twl6030_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, TWL6030_TOGGLE3_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to disable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to disable PWM\n", pwm->label); goto out; } @@ -262,7 +261,7 @@ static void twl6030_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, TWL6030_TOGGLE3_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to disable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to disable PWM\n", pwm->label); goto out; } @@ -270,7 +269,7 @@ static void twl6030_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, TWL6030_TOGGLE3_REG); if (ret < 0) { - dev_err(chip->dev, "%s: Failed to disable PWM\n", pwm->label); + dev_err(pwmchip_parent(chip), "%s: Failed to disable PWM\n", pwm->label); goto out; } @@ -341,23 +340,22 @@ static const struct pwm_ops twl6030_pwm_ops = { static int twl_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct twl_pwm_chip *twl; - twl = devm_kzalloc(&pdev->dev, sizeof(*twl), GFP_KERNEL); - if (!twl) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, 2, sizeof(*twl)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + twl = to_twl(chip); if (twl_class_is_4030()) - twl->chip.ops = &twl4030_pwm_ops; + chip->ops = &twl4030_pwm_ops; else - twl->chip.ops = &twl6030_pwm_ops; - - twl->chip.dev = &pdev->dev; - twl->chip.npwm = 2; + chip->ops = &twl6030_pwm_ops; mutex_init(&twl->mutex); - return devm_pwmchip_add(&pdev->dev, &twl->chip); + return devm_pwmchip_add(&pdev->dev, chip); } #ifdef CONFIG_OF diff --git a/drivers/pwm/pwm-visconti.c b/drivers/pwm/pwm-visconti.c index 8d736d558122..9e55380957be 100644 --- a/drivers/pwm/pwm-visconti.c +++ b/drivers/pwm/pwm-visconti.c @@ -34,13 +34,12 @@ #define PIPGM_PWMC_POLARITY_MASK GENMASK(5, 5) struct visconti_pwm_chip { - struct pwm_chip chip; void __iomem *base; }; static inline struct visconti_pwm_chip *visconti_pwm_from_chip(struct pwm_chip *chip) { - return container_of(chip, struct visconti_pwm_chip, chip); + return pwmchip_get_drvdata(chip); } static int visconti_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, @@ -134,22 +133,22 @@ static const struct pwm_ops visconti_pwm_ops = { static int visconti_pwm_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + struct pwm_chip *chip; struct visconti_pwm_chip *priv; int ret; - priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; + chip = devm_pwmchip_alloc(dev, 4, sizeof(*priv)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + priv = visconti_pwm_from_chip(chip); priv->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(priv->base)) return PTR_ERR(priv->base); - priv->chip.dev = dev; - priv->chip.ops = &visconti_pwm_ops; - priv->chip.npwm = 4; + chip->ops = &visconti_pwm_ops; - ret = devm_pwmchip_add(&pdev->dev, &priv->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "Cannot register visconti PWM\n"); diff --git a/drivers/pwm/pwm-vt8500.c b/drivers/pwm/pwm-vt8500.c index 7bfeacee05d0..016c82d65527 100644 --- a/drivers/pwm/pwm-vt8500.c +++ b/drivers/pwm/pwm-vt8500.c @@ -45,16 +45,19 @@ #define STATUS_ALL_UPDATE 0x0F struct vt8500_chip { - struct pwm_chip chip; void __iomem *base; struct clk *clk; }; -#define to_vt8500_chip(chip) container_of(chip, struct vt8500_chip, chip) +static inline struct vt8500_chip *to_vt8500_chip(struct pwm_chip *chip) +{ + return pwmchip_get_drvdata(chip); +} #define msecs_to_loops(t) (loops_per_jiffy / 1000 * HZ * t) -static inline void vt8500_pwm_busy_wait(struct vt8500_chip *vt8500, int nr, u8 bitmask) +static inline void vt8500_pwm_busy_wait(struct pwm_chip *chip, int nr, u8 bitmask) { + struct vt8500_chip *vt8500 = to_vt8500_chip(chip); int loops = msecs_to_loops(10); u32 mask = bitmask << (nr << 8); @@ -62,7 +65,7 @@ static inline void vt8500_pwm_busy_wait(struct vt8500_chip *vt8500, int nr, u8 b cpu_relax(); if (unlikely(!loops)) - dev_warn(vt8500->chip.dev, "Waiting for status bits 0x%x to clear timed out\n", + dev_warn(pwmchip_parent(chip), "Waiting for status bits 0x%x to clear timed out\n", mask); } @@ -77,7 +80,7 @@ static int vt8500_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, err = clk_enable(vt8500->clk); if (err < 0) { - dev_err(chip->dev, "failed to enable clock\n"); + dev_err(pwmchip_parent(chip), "failed to enable clock\n"); return err; } @@ -103,18 +106,18 @@ static int vt8500_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, dc = div64_u64(c, period_ns); writel(prescale, vt8500->base + REG_SCALAR(pwm->hwpwm)); - vt8500_pwm_busy_wait(vt8500, pwm->hwpwm, STATUS_SCALAR_UPDATE); + vt8500_pwm_busy_wait(chip, pwm->hwpwm, STATUS_SCALAR_UPDATE); writel(pv, vt8500->base + REG_PERIOD(pwm->hwpwm)); - vt8500_pwm_busy_wait(vt8500, pwm->hwpwm, STATUS_PERIOD_UPDATE); + vt8500_pwm_busy_wait(chip, pwm->hwpwm, STATUS_PERIOD_UPDATE); writel(dc, vt8500->base + REG_DUTY(pwm->hwpwm)); - vt8500_pwm_busy_wait(vt8500, pwm->hwpwm, STATUS_DUTY_UPDATE); + vt8500_pwm_busy_wait(chip, pwm->hwpwm, STATUS_DUTY_UPDATE); val = readl(vt8500->base + REG_CTRL(pwm->hwpwm)); val |= CTRL_AUTOLOAD; writel(val, vt8500->base + REG_CTRL(pwm->hwpwm)); - vt8500_pwm_busy_wait(vt8500, pwm->hwpwm, STATUS_CTRL_UPDATE); + vt8500_pwm_busy_wait(chip, pwm->hwpwm, STATUS_CTRL_UPDATE); clk_disable(vt8500->clk); return 0; @@ -128,14 +131,14 @@ static int vt8500_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm) err = clk_enable(vt8500->clk); if (err < 0) { - dev_err(chip->dev, "failed to enable clock\n"); + dev_err(pwmchip_parent(chip), "failed to enable clock\n"); return err; } val = readl(vt8500->base + REG_CTRL(pwm->hwpwm)); val |= CTRL_ENABLE; writel(val, vt8500->base + REG_CTRL(pwm->hwpwm)); - vt8500_pwm_busy_wait(vt8500, pwm->hwpwm, STATUS_CTRL_UPDATE); + vt8500_pwm_busy_wait(chip, pwm->hwpwm, STATUS_CTRL_UPDATE); return 0; } @@ -148,7 +151,7 @@ static void vt8500_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm) val = readl(vt8500->base + REG_CTRL(pwm->hwpwm)); val &= ~CTRL_ENABLE; writel(val, vt8500->base + REG_CTRL(pwm->hwpwm)); - vt8500_pwm_busy_wait(vt8500, pwm->hwpwm, STATUS_CTRL_UPDATE); + vt8500_pwm_busy_wait(chip, pwm->hwpwm, STATUS_CTRL_UPDATE); clk_disable(vt8500->clk); } @@ -168,7 +171,7 @@ static int vt8500_pwm_set_polarity(struct pwm_chip *chip, val &= ~CTRL_INVERT; writel(val, vt8500->base + REG_CTRL(pwm->hwpwm)); - vt8500_pwm_busy_wait(vt8500, pwm->hwpwm, STATUS_CTRL_UPDATE); + vt8500_pwm_busy_wait(chip, pwm->hwpwm, STATUS_CTRL_UPDATE); return 0; } @@ -231,6 +234,7 @@ MODULE_DEVICE_TABLE(of, vt8500_pwm_dt_ids); static int vt8500_pwm_probe(struct platform_device *pdev) { + struct pwm_chip *chip; struct vt8500_chip *vt8500; struct device_node *np = pdev->dev.of_node; int ret; @@ -238,13 +242,12 @@ static int vt8500_pwm_probe(struct platform_device *pdev) if (!np) return dev_err_probe(&pdev->dev, -EINVAL, "invalid devicetree node\n"); - vt8500 = devm_kzalloc(&pdev->dev, sizeof(*vt8500), GFP_KERNEL); - if (vt8500 == NULL) - return -ENOMEM; + chip = devm_pwmchip_alloc(&pdev->dev, VT8500_NR_PWMS, sizeof(*vt8500)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + vt8500 = to_vt8500_chip(chip); - vt8500->chip.dev = &pdev->dev; - vt8500->chip.ops = &vt8500_pwm_ops; - vt8500->chip.npwm = VT8500_NR_PWMS; + chip->ops = &vt8500_pwm_ops; vt8500->clk = devm_clk_get_prepared(&pdev->dev, NULL); if (IS_ERR(vt8500->clk)) @@ -254,7 +257,7 @@ static int vt8500_pwm_probe(struct platform_device *pdev) if (IS_ERR(vt8500->base)) return PTR_ERR(vt8500->base); - ret = devm_pwmchip_add(&pdev->dev, &vt8500->chip); + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "failed to add PWM chip\n"); diff --git a/drivers/pwm/pwm-xilinx.c b/drivers/pwm/pwm-xilinx.c index 5f3c2a6fed11..3a7deebb0d0c 100644 --- a/drivers/pwm/pwm-xilinx.c +++ b/drivers/pwm/pwm-xilinx.c @@ -80,15 +80,10 @@ unsigned int xilinx_timer_get_period(struct xilinx_timer_priv *priv, #define TCSR_PWM_CLEAR (TCSR_MDT | TCSR_LOAD) #define TCSR_PWM_MASK (TCSR_PWM_SET | TCSR_PWM_CLEAR) -struct xilinx_pwm_device { - struct pwm_chip chip; - struct xilinx_timer_priv priv; -}; - static inline struct xilinx_timer_priv *xilinx_pwm_chip_to_priv(struct pwm_chip *chip) { - return &container_of(chip, struct xilinx_pwm_device, chip)->priv; + return pwmchip_get_drvdata(chip); } static bool xilinx_timer_pwm_enabled(u32 tcsr0, u32 tcsr1) @@ -214,7 +209,7 @@ static int xilinx_pwm_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; struct xilinx_timer_priv *priv; - struct xilinx_pwm_device *xilinx_pwm; + struct pwm_chip *chip; u32 pwm_cells, one_timer, width; void __iomem *regs; @@ -225,11 +220,11 @@ static int xilinx_pwm_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "could not read #pwm-cells\n"); - xilinx_pwm = devm_kzalloc(dev, sizeof(*xilinx_pwm), GFP_KERNEL); - if (!xilinx_pwm) - return -ENOMEM; - platform_set_drvdata(pdev, xilinx_pwm); - priv = &xilinx_pwm->priv; + chip = devm_pwmchip_alloc(dev, 1, sizeof(*priv)); + if (IS_ERR(chip)) + return PTR_ERR(chip); + priv = xilinx_pwm_chip_to_priv(chip); + platform_set_drvdata(pdev, chip); regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(regs)) @@ -278,10 +273,8 @@ static int xilinx_pwm_probe(struct platform_device *pdev) return dev_err_probe(dev, ret, "Clock enable failed\n"); clk_rate_exclusive_get(priv->clk); - xilinx_pwm->chip.dev = dev; - xilinx_pwm->chip.ops = &xilinx_pwm_ops; - xilinx_pwm->chip.npwm = 1; - ret = pwmchip_add(&xilinx_pwm->chip); + chip->ops = &xilinx_pwm_ops; + ret = pwmchip_add(chip); if (ret) { clk_rate_exclusive_put(priv->clk); clk_disable_unprepare(priv->clk); @@ -293,11 +286,12 @@ static int xilinx_pwm_probe(struct platform_device *pdev) static void xilinx_pwm_remove(struct platform_device *pdev) { - struct xilinx_pwm_device *xilinx_pwm = platform_get_drvdata(pdev); + struct pwm_chip *chip = platform_get_drvdata(pdev); + struct xilinx_timer_priv *priv = xilinx_pwm_chip_to_priv(chip); - pwmchip_remove(&xilinx_pwm->chip); - clk_rate_exclusive_put(xilinx_pwm->priv.clk); - clk_disable_unprepare(xilinx_pwm->priv.clk); + pwmchip_remove(chip); + clk_rate_exclusive_put(priv->clk); + clk_disable_unprepare(priv->clk); } static const struct of_device_id xilinx_pwm_of_match[] = { diff --git a/drivers/pwm/sysfs.c b/drivers/pwm/sysfs.c index 1698609d91c8..3f434a771fb5 100644 --- a/drivers/pwm/sysfs.c +++ b/drivers/pwm/sysfs.c @@ -509,10 +509,10 @@ void pwmchip_sysfs_export(struct pwm_chip *chip) * If device_create() fails the pwm_chip is still usable by * the kernel it's just not exported. */ - parent = device_create(&pwm_class, chip->dev, MKDEV(0, 0), chip, + parent = device_create(&pwm_class, pwmchip_parent(chip), MKDEV(0, 0), chip, "pwmchip%d", chip->id); if (IS_ERR(parent)) { - dev_warn(chip->dev, + dev_warn(pwmchip_parent(chip), "device_create failed for pwm_chip sysfs export\n"); } } diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c index a968dabb48f5..d019ca6dee9b 100644 --- a/drivers/regulator/core.c +++ b/drivers/regulator/core.c @@ -3932,7 +3932,6 @@ static int regulator_get_optimal_voltage(struct regulator_dev *rdev, if (ret < 0) return ret; - possible_uV = desired_min_uV; done = true; goto finish; @@ -5891,7 +5890,7 @@ static const struct dev_pm_ops __maybe_unused regulator_pm_ops = { }; #endif -struct class regulator_class = { +const struct class regulator_class = { .name = "regulator", .dev_release = regulator_dev_release, .dev_groups = regulator_dev_groups, diff --git a/drivers/regulator/da9055-regulator.c b/drivers/regulator/da9055-regulator.c index 8fd9ac787588..352547c375bd 100644 --- a/drivers/regulator/da9055-regulator.c +++ b/drivers/regulator/da9055-regulator.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -413,31 +412,35 @@ static struct da9055_regulator_info da9055_regulator_info[] = { * GPIO can control regulator state and/or select the regulator register * set A/B for voltage ramping. */ -static int da9055_gpio_init(struct da9055_regulator *regulator, +static int da9055_gpio_init(struct device *dev, + struct da9055_regulator *regulator, struct regulator_config *config, struct da9055_pdata *pdata, int id) { struct da9055_regulator_info *info = regulator->info; + struct gpio_desc *ren; + struct gpio_desc *ena; + struct gpio_desc *rsel; int ret = 0; - if (!pdata) - return 0; + /* Look for "regulator-enable-gpios" GPIOs in the regulator node */ + ren = devm_gpiod_get_optional(dev, "regulator-enable", GPIOD_IN); + if (IS_ERR(ren)) + return PTR_ERR(ren); - if (pdata->gpio_ren && pdata->gpio_ren[id]) { - char name[18]; - int gpio_mux = pdata->gpio_ren[id]; + if (ren) { + /* This GPIO is not optional at this point */ + ena = devm_gpiod_get(dev, "enable", GPIOD_OUT_HIGH); + if (IS_ERR(ena)) + return PTR_ERR(ena); - config->ena_gpiod = pdata->ena_gpiods[id]; + config->ena_gpiod = ena; /* * GPI pin is muxed with regulator to control the * regulator state. */ - sprintf(name, "DA9055 GPI %d", gpio_mux); - ret = devm_gpio_request_one(config->dev, gpio_mux, GPIOF_DIR_IN, - name); - if (ret < 0) - goto err; + gpiod_set_consumer_name(ren, "DA9055 ren GPI"); /* * Let the regulator know that its state is controlled @@ -448,24 +451,22 @@ static int da9055_gpio_init(struct da9055_regulator *regulator, pdata->reg_ren[id] << DA9055_E_GPI_SHIFT); if (ret < 0) - goto err; + return ret; } - if (pdata->gpio_rsel && pdata->gpio_rsel[id]) { - char name[18]; - int gpio_mux = pdata->gpio_rsel[id]; + /* Look for "regulator-select-gpios" GPIOs in the regulator node */ + rsel = devm_gpiod_get_optional(dev, "regulator-select", GPIOD_IN); + if (IS_ERR(rsel)) + return PTR_ERR(rsel); + if (rsel) { regulator->reg_rselect = pdata->reg_rsel[id]; /* * GPI pin is muxed with regulator to select the * regulator register set A/B for voltage ramping. */ - sprintf(name, "DA9055 GPI %d", gpio_mux); - ret = devm_gpio_request_one(config->dev, gpio_mux, GPIOF_DIR_IN, - name); - if (ret < 0) - goto err; + gpiod_set_consumer_name(rsel, "DA9055 rsel GPI"); /* * Let the regulator know that its register set A/B @@ -477,7 +478,6 @@ static int da9055_gpio_init(struct da9055_regulator *regulator, << DA9055_V_GPI_SHIFT); } -err: return ret; } @@ -532,7 +532,7 @@ static int da9055_regulator_probe(struct platform_device *pdev) if (pdata) config.init_data = pdata->regulators[pdev->id]; - ret = da9055_gpio_init(regulator, &config, pdata, pdev->id); + ret = da9055_gpio_init(&pdev->dev, regulator, &config, pdata, pdev->id); if (ret < 0) return ret; diff --git a/drivers/regulator/da9121-regulator.c b/drivers/regulator/da9121-regulator.c index b551a400bdd1..5ee76b533576 100644 --- a/drivers/regulator/da9121-regulator.c +++ b/drivers/regulator/da9121-regulator.c @@ -14,7 +14,6 @@ // Copyright (C) 2020 Dialog Semiconductor #include -#include #include #include #include diff --git a/drivers/regulator/fixed-helper.c b/drivers/regulator/fixed-helper.c index 0eb2442456f0..2d5a42b2b3d8 100644 --- a/drivers/regulator/fixed-helper.c +++ b/drivers/regulator/fixed-helper.c @@ -15,7 +15,7 @@ static void regulator_fixed_release(struct device *dev) { struct fixed_regulator_data *data = container_of(dev, struct fixed_regulator_data, pdev.dev); - kfree(data->cfg.supply_name); + kfree_const(data->cfg.supply_name); kfree(data); } @@ -36,7 +36,7 @@ struct platform_device *regulator_register_always_on(int id, const char *name, if (!data) return NULL; - data->cfg.supply_name = kstrdup(name, GFP_KERNEL); + data->cfg.supply_name = kstrdup_const(name, GFP_KERNEL); if (!data->cfg.supply_name) { kfree(data); return NULL; diff --git a/drivers/regulator/internal.h b/drivers/regulator/internal.h index fb4433068d29..77a502141089 100644 --- a/drivers/regulator/internal.h +++ b/drivers/regulator/internal.h @@ -58,7 +58,7 @@ struct regulator { struct dentry *debugfs; }; -extern struct class regulator_class; +extern const struct class regulator_class; static inline struct regulator_dev *dev_to_rdev(struct device *dev) { diff --git a/drivers/regulator/lp873x-regulator.c b/drivers/regulator/lp873x-regulator.c index 8dfdd1db2070..84a134cfcd9c 100644 --- a/drivers/regulator/lp873x-regulator.c +++ b/drivers/regulator/lp873x-regulator.c @@ -5,6 +5,7 @@ * Copyright (C) 2016 Texas Instruments Incorporated - https://www.ti.com/ */ +#include #include #include #include @@ -93,7 +94,7 @@ static int lp873x_buck_set_ramp_delay(struct regulator_dev *rdev, ret = regmap_update_bits(lp873->regmap, regulators[id].ctrl2_reg, LP873X_BUCK0_CTRL_2_BUCK0_SLEW_RATE, - reg << __ffs(LP873X_BUCK0_CTRL_2_BUCK0_SLEW_RATE)); + FIELD_PREP(LP873X_BUCK0_CTRL_2_BUCK0_SLEW_RATE, reg)); if (ret) { dev_err(lp873->dev, "SLEW RATE write failed: %d\n", ret); return ret; diff --git a/drivers/regulator/lp87565-regulator.c b/drivers/regulator/lp87565-regulator.c index 61ee5cf3f241..1259b5d20153 100644 --- a/drivers/regulator/lp87565-regulator.c +++ b/drivers/regulator/lp87565-regulator.c @@ -5,6 +5,7 @@ * Copyright (C) 2017 Texas Instruments Incorporated - https://www.ti.com/ */ +#include #include #include #include @@ -99,7 +100,7 @@ static int lp87565_buck_set_ramp_delay(struct regulator_dev *rdev, ret = regmap_update_bits(rdev->regmap, regulators[id].ctrl2_reg, LP87565_BUCK_CTRL_2_SLEW_RATE, - reg << __ffs(LP87565_BUCK_CTRL_2_SLEW_RATE)); + FIELD_PREP(LP87565_BUCK_CTRL_2_SLEW_RATE, reg)); if (ret) { dev_err(&rdev->dev, "SLEW RATE write failed: %d\n", ret); return ret; diff --git a/drivers/regulator/lp8788-buck.c b/drivers/regulator/lp8788-buck.c index e97ade09dede..2ade249ab6df 100644 --- a/drivers/regulator/lp8788-buck.c +++ b/drivers/regulator/lp8788-buck.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include /* register address */ #define LP8788_EN_BUCK 0x0C @@ -69,8 +69,8 @@ #define BUCK_FPWM_SHIFT(x) (x) enum lp8788_dvs_state { - DVS_LOW = GPIOF_OUT_INIT_LOW, - DVS_HIGH = GPIOF_OUT_INIT_HIGH, + DVS_LOW = 0, + DVS_HIGH = 1, }; enum lp8788_dvs_mode { @@ -89,6 +89,8 @@ struct lp8788_buck { struct lp8788 *lp; struct regulator_dev *regulator; void *dvs; + struct gpio_desc *gpio1; + struct gpio_desc *gpio2; /* Only used on BUCK2 */ }; /* BUCK 1 ~ 4 voltage ranges */ @@ -106,8 +108,7 @@ static void lp8788_buck1_set_dvs(struct lp8788_buck *buck) return; pinstate = dvs->vsel == DVS_SEL_V0 ? DVS_LOW : DVS_HIGH; - if (gpio_is_valid(dvs->gpio)) - gpio_set_value(dvs->gpio, pinstate); + gpiod_set_value(buck->gpio1, pinstate); } static void lp8788_buck2_set_dvs(struct lp8788_buck *buck) @@ -139,11 +140,8 @@ static void lp8788_buck2_set_dvs(struct lp8788_buck *buck) return; } - if (gpio_is_valid(dvs->gpio[0])) - gpio_set_value(dvs->gpio[0], pin1); - - if (gpio_is_valid(dvs->gpio[1])) - gpio_set_value(dvs->gpio[1], pin2); + gpiod_set_value(buck->gpio1, pin1); + gpiod_set_value(buck->gpio2, pin2); } static void lp8788_set_dvs(struct lp8788_buck *buck, enum lp8788_buck_id id) @@ -202,19 +200,13 @@ static u8 lp8788_select_buck_vout_addr(struct lp8788_buck *buck, enum lp8788_buck_id id) { enum lp8788_dvs_mode mode = lp8788_get_buck_dvs_ctrl_mode(buck, id); - struct lp8788_buck1_dvs *b1_dvs; - struct lp8788_buck2_dvs *b2_dvs; u8 val, idx, addr; int pin1, pin2; switch (id) { case BUCK1: if (mode == EXTPIN) { - b1_dvs = (struct lp8788_buck1_dvs *)buck->dvs; - if (!b1_dvs) - goto err; - - idx = gpio_get_value(b1_dvs->gpio) ? 1 : 0; + idx = gpiod_get_value(buck->gpio1); } else { lp8788_read_byte(buck->lp, LP8788_BUCK_DVS_SEL, &val); idx = (val & LP8788_BUCK1_DVS_M) >> LP8788_BUCK1_DVS_S; @@ -223,12 +215,8 @@ static u8 lp8788_select_buck_vout_addr(struct lp8788_buck *buck, break; case BUCK2: if (mode == EXTPIN) { - b2_dvs = (struct lp8788_buck2_dvs *)buck->dvs; - if (!b2_dvs) - goto err; - - pin1 = gpio_get_value(b2_dvs->gpio[0]); - pin2 = gpio_get_value(b2_dvs->gpio[1]); + pin1 = gpiod_get_value(buck->gpio1); + pin2 = gpiod_get_value(buck->gpio2); if (pin1 == PIN_LOW && pin2 == PIN_LOW) idx = 0; @@ -424,28 +412,28 @@ static int lp8788_dvs_gpio_request(struct platform_device *pdev, enum lp8788_buck_id id) { struct lp8788_platform_data *pdata = buck->lp->pdata; - char *b1_name = "LP8788_B1_DVS"; - char *b2_name[] = { "LP8788_B2_DVS1", "LP8788_B2_DVS2" }; - int i, gpio, ret; + struct device *dev = &pdev->dev; switch (id) { case BUCK1: - gpio = pdata->buck1_dvs->gpio; - ret = devm_gpio_request_one(&pdev->dev, gpio, DVS_LOW, - b1_name); - if (ret) - return ret; + buck->gpio1 = devm_gpiod_get(dev, "dvs", GPIOD_OUT_LOW); + if (IS_ERR(buck->gpio1)) + return PTR_ERR(buck->gpio1); + gpiod_set_consumer_name(buck->gpio1, "LP8788_B1_DVS"); buck->dvs = pdata->buck1_dvs; break; case BUCK2: - for (i = 0; i < LP8788_NUM_BUCK2_DVS; i++) { - gpio = pdata->buck2_dvs->gpio[i]; - ret = devm_gpio_request_one(&pdev->dev, gpio, - DVS_LOW, b2_name[i]); - if (ret) - return ret; - } + buck->gpio1 = devm_gpiod_get_index(dev, "dvs", 0, GPIOD_OUT_LOW); + if (IS_ERR(buck->gpio1)) + return PTR_ERR(buck->gpio1); + gpiod_set_consumer_name(buck->gpio1, "LP8788_B2_DVS1"); + + buck->gpio2 = devm_gpiod_get_index(dev, "dvs", 1, GPIOD_OUT_LOW); + if (IS_ERR(buck->gpio2)) + return PTR_ERR(buck->gpio2); + gpiod_set_consumer_name(buck->gpio2, "LP8788_B2_DVS2"); + buck->dvs = pdata->buck2_dvs; break; default: diff --git a/drivers/regulator/max8973-regulator.c b/drivers/regulator/max8973-regulator.c index 8d5193207552..f8bb6828feef 100644 --- a/drivers/regulator/max8973-regulator.c +++ b/drivers/regulator/max8973-regulator.c @@ -20,9 +20,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -102,7 +100,7 @@ struct max8973_chip { struct regulator_desc desc; struct regmap *regmap; bool enable_external_control; - int dvs_gpio; + struct gpio_desc *dvs_gpiod; int lru_index[MAX8973_MAX_VOUT_REG]; int curr_vout_val[MAX8973_MAX_VOUT_REG]; int curr_vout_reg; @@ -184,7 +182,7 @@ static int max8973_dcdc_set_voltage_sel(struct regulator_dev *rdev, * If gpios are available to select the VOUT register then least * recently used register for new configuration. */ - if (gpio_is_valid(max->dvs_gpio)) + if (max->dvs_gpiod) found = find_voltage_set_register(max, vsel, &vout_reg, &gpio_val); @@ -201,8 +199,8 @@ static int max8973_dcdc_set_voltage_sel(struct regulator_dev *rdev, } /* Select proper VOUT register vio gpios */ - if (gpio_is_valid(max->dvs_gpio)) { - gpio_set_value_cansleep(max->dvs_gpio, gpio_val & 0x1); + if (max->dvs_gpiod) { + gpiod_set_value_cansleep(max->dvs_gpiod, gpio_val & 0x1); max->curr_gpio_val = gpio_val; } return 0; @@ -531,7 +529,6 @@ static struct max8973_regulator_platform_data *max8973_parse_dt( pdata->enable_ext_control = of_property_read_bool(np, "maxim,externally-enable"); - pdata->dvs_gpio = of_get_named_gpio(np, "maxim,dvs-gpio", 0); ret = of_property_read_u32(np, "maxim,dvs-default-state", &pval); if (!ret) @@ -612,13 +609,17 @@ static int max8973_probe(struct i2c_client *client) return -EIO; } - if (pdata->dvs_gpio == -EPROBE_DEFER) - return -EPROBE_DEFER; - max = devm_kzalloc(&client->dev, sizeof(*max), GFP_KERNEL); if (!max) return -ENOMEM; + max->dvs_gpiod = devm_gpiod_get_optional(&client->dev, "maxim,dvs", + (pdata->dvs_def_state) ? GPIOD_OUT_HIGH : GPIOD_OUT_LOW); + if (IS_ERR(max->dvs_gpiod)) + return dev_err_probe(&client->dev, PTR_ERR(max->dvs_gpiod), + "failed to obtain dvs gpio\n"); + gpiod_set_consumer_name(max->dvs_gpiod, "max8973-dvs"); + max->regmap = devm_regmap_init_i2c(client, &max8973_regmap_config); if (IS_ERR(max->regmap)) { ret = PTR_ERR(max->regmap); @@ -663,7 +664,6 @@ static int max8973_probe(struct i2c_client *client) max->desc.ramp_delay_table = max8973_buck_ramp_table; max->desc.n_ramp_values = ARRAY_SIZE(max8973_buck_ramp_table); - max->dvs_gpio = (pdata->dvs_gpio) ? pdata->dvs_gpio : -EINVAL; max->enable_external_control = pdata->enable_ext_control; max->curr_gpio_val = pdata->dvs_def_state; max->curr_vout_reg = MAX8973_VOUT + pdata->dvs_def_state; @@ -671,21 +671,9 @@ static int max8973_probe(struct i2c_client *client) max->lru_index[0] = max->curr_vout_reg; - if (gpio_is_valid(max->dvs_gpio)) { - int gpio_flags; + if (max->dvs_gpiod) { int i; - gpio_flags = (pdata->dvs_def_state) ? - GPIOF_OUT_INIT_HIGH : GPIOF_OUT_INIT_LOW; - ret = devm_gpio_request_one(&client->dev, max->dvs_gpio, - gpio_flags, "max8973-dvs"); - if (ret) { - dev_err(&client->dev, - "gpio_request for gpio %d failed, err = %d\n", - max->dvs_gpio, ret); - return ret; - } - /* * Initialize the lru index with vout_reg id * The index 0 will be most recently used and diff --git a/drivers/regulator/max8997-regulator.c b/drivers/regulator/max8997-regulator.c index 0b38eaa73597..5f201ee9a5b8 100644 --- a/drivers/regulator/max8997-regulator.c +++ b/drivers/regulator/max8997-regulator.c @@ -9,8 +9,7 @@ #include #include -#include -#include +#include #include #include #include @@ -32,7 +31,7 @@ struct max8997_data { u8 buck1_vol[8]; u8 buck2_vol[8]; u8 buck5_vol[8]; - int buck125_gpios[3]; + struct gpio_desc *buck125_gpiods[3]; int buck125_gpioindex; bool ignore_gpiodvs_side_effect; @@ -52,9 +51,9 @@ static inline void max8997_set_gpio(struct max8997_data *max8997) int set2 = ((max8997->buck125_gpioindex) >> 1) & 0x1; int set1 = ((max8997->buck125_gpioindex) >> 2) & 0x1; - gpio_set_value(max8997->buck125_gpios[0], set1); - gpio_set_value(max8997->buck125_gpios[1], set2); - gpio_set_value(max8997->buck125_gpios[2], set3); + gpiod_set_value(max8997->buck125_gpiods[0], set1); + gpiod_set_value(max8997->buck125_gpiods[1], set2); + gpiod_set_value(max8997->buck125_gpiods[2], set3); } struct voltage_map_desc { @@ -873,31 +872,13 @@ static struct regulator_desc regulators[] = { }; #ifdef CONFIG_OF -static int max8997_pmic_dt_parse_dvs_gpio(struct platform_device *pdev, - struct max8997_platform_data *pdata, - struct device_node *pmic_np) -{ - int i, gpio; - - for (i = 0; i < 3; i++) { - gpio = of_get_named_gpio(pmic_np, - "max8997,pmic-buck125-dvs-gpios", i); - if (!gpio_is_valid(gpio)) { - dev_err(&pdev->dev, "invalid gpio[%d]: %d\n", i, gpio); - return -EINVAL; - } - pdata->buck125_gpios[i] = gpio; - } - return 0; -} - static int max8997_pmic_dt_parse_pdata(struct platform_device *pdev, struct max8997_platform_data *pdata) { struct max8997_dev *iodev = dev_get_drvdata(pdev->dev.parent); struct device_node *pmic_np, *regulators_np, *reg_np; struct max8997_regulator_data *rdata; - unsigned int i, dvs_voltage_nr = 1, ret; + unsigned int i, dvs_voltage_nr = 1; pmic_np = iodev->dev->of_node; if (!pmic_np) { @@ -949,10 +930,6 @@ static int max8997_pmic_dt_parse_pdata(struct platform_device *pdev, if (pdata->buck1_gpiodvs || pdata->buck2_gpiodvs || pdata->buck5_gpiodvs) { - ret = max8997_pmic_dt_parse_dvs_gpio(pdev, pdata, pmic_np); - if (ret) - return -EINVAL; - if (of_property_read_u32(pmic_np, "max8997,pmic-buck125-default-dvs-idx", &pdata->buck125_default_idx)) { @@ -1039,7 +1016,6 @@ static int max8997_pmic_probe(struct platform_device *pdev) max8997->buck1_gpiodvs = pdata->buck1_gpiodvs; max8997->buck2_gpiodvs = pdata->buck2_gpiodvs; max8997->buck5_gpiodvs = pdata->buck5_gpiodvs; - memcpy(max8997->buck125_gpios, pdata->buck125_gpios, sizeof(int) * 3); max8997->ignore_gpiodvs_side_effect = pdata->ignore_gpiodvs_side_effect; nr_dvs = (pdata->buck1_gpiodvs || pdata->buck2_gpiodvs || @@ -1110,38 +1086,27 @@ static int max8997_pmic_probe(struct platform_device *pdev) */ if (pdata->buck1_gpiodvs || pdata->buck2_gpiodvs || pdata->buck5_gpiodvs) { + const char *gpio_names[3] = {"MAX8997 SET1", "MAX8997 SET2", "MAX8997 SET3"}; - if (!gpio_is_valid(pdata->buck125_gpios[0]) || - !gpio_is_valid(pdata->buck125_gpios[1]) || - !gpio_is_valid(pdata->buck125_gpios[2])) { - dev_err(&pdev->dev, "GPIO NOT VALID\n"); - return -EINVAL; + for (i = 0; i < 3; i++) { + enum gpiod_flags flags; + + if (max8997->buck125_gpioindex & BIT(2 - i)) + flags = GPIOD_OUT_HIGH; + else + flags = GPIOD_OUT_LOW; + + max8997->buck125_gpiods[i] = devm_gpiod_get_index(iodev->dev, + "max8997,pmic-buck125-dvs", + i, + flags); + if (IS_ERR(max8997->buck125_gpiods[i])) { + ret = PTR_ERR(max8997->buck125_gpiods[i]); + return dev_err_probe(iodev->dev, ret, "cant get GPIO %d (%d)\n", + i, ret); + } + gpiod_set_consumer_name(max8997->buck125_gpiods[i], gpio_names[i]); } - - ret = devm_gpio_request(&pdev->dev, pdata->buck125_gpios[0], - "MAX8997 SET1"); - if (ret) - return ret; - - ret = devm_gpio_request(&pdev->dev, pdata->buck125_gpios[1], - "MAX8997 SET2"); - if (ret) - return ret; - - ret = devm_gpio_request(&pdev->dev, pdata->buck125_gpios[2], - "MAX8997 SET3"); - if (ret) - return ret; - - gpio_direction_output(pdata->buck125_gpios[0], - (max8997->buck125_gpioindex >> 2) - & 0x1); /* SET1 */ - gpio_direction_output(pdata->buck125_gpios[1], - (max8997->buck125_gpioindex >> 1) - & 0x1); /* SET2 */ - gpio_direction_output(pdata->buck125_gpios[2], - (max8997->buck125_gpioindex >> 0) - & 0x1); /* SET3 */ } /* DVS-GPIO disabled */ diff --git a/drivers/regulator/max8998.c b/drivers/regulator/max8998.c index fadb4717384a..254a77887f66 100644 --- a/drivers/regulator/max8998.c +++ b/drivers/regulator/max8998.c @@ -10,12 +10,12 @@ #include #include #include -#include +#include +#include #include #include #include #include -#include #include #include #include @@ -31,6 +31,9 @@ struct max8998_data { unsigned int buck1_idx; /* index to last changed voltage */ /* value in a set */ unsigned int buck2_idx; + struct gpio_desc *buck1_gpio1; + struct gpio_desc *buck1_gpio2; + struct gpio_desc *buck2_gpio; }; static const unsigned int charger_current_table[] = { @@ -227,15 +230,15 @@ static int max8998_set_voltage_ldo_sel(struct regulator_dev *rdev, return ret; } -static inline void buck1_gpio_set(int gpio1, int gpio2, int v) +static inline void buck1_gpio_set(struct gpio_desc *gpio1, struct gpio_desc *gpio2, int v) { - gpio_set_value(gpio1, v & 0x1); - gpio_set_value(gpio2, (v >> 1) & 0x1); + gpiod_set_value(gpio1, v & 0x1); + gpiod_set_value(gpio2, (v >> 1) & 0x1); } -static inline void buck2_gpio_set(int gpio, int v) +static inline void buck2_gpio_set(struct gpio_desc *gpio, int v) { - gpio_set_value(gpio, v & 0x1); + gpiod_set_value(gpio, v & 0x1); } static int max8998_set_voltage_buck_sel(struct regulator_dev *rdev, @@ -260,16 +263,15 @@ static int max8998_set_voltage_buck_sel(struct regulator_dev *rdev, selector, max8998->buck1_vol[0], max8998->buck1_vol[1], max8998->buck1_vol[2], max8998->buck1_vol[3]); - if (gpio_is_valid(pdata->buck1_set1) && - gpio_is_valid(pdata->buck1_set2)) { + if (max8998->buck1_gpio1 && max8998->buck1_gpio2) { /* check if requested voltage */ /* value is already defined */ for (j = 0; j < ARRAY_SIZE(max8998->buck1_vol); j++) { if (max8998->buck1_vol[j] == selector) { max8998->buck1_idx = j; - buck1_gpio_set(pdata->buck1_set1, - pdata->buck1_set2, j); + buck1_gpio_set(max8998->buck1_gpio1, + max8998->buck1_gpio2, j); goto buck1_exit; } } @@ -286,13 +288,13 @@ static int max8998_set_voltage_buck_sel(struct regulator_dev *rdev, &shift, &mask); ret = max8998_write_reg(i2c, reg, selector); - buck1_gpio_set(pdata->buck1_set1, - pdata->buck1_set2, max8998->buck1_idx); + buck1_gpio_set(max8998->buck1_gpio1, + max8998->buck1_gpio2, max8998->buck1_idx); buck1_last_val++; buck1_exit: dev_dbg(max8998->dev, "%s: SET1:%d, SET2:%d\n", - i2c->name, gpio_get_value(pdata->buck1_set1), - gpio_get_value(pdata->buck1_set2)); + i2c->name, gpiod_get_value(max8998->buck1_gpio1), + gpiod_get_value(max8998->buck1_gpio2)); break; } else { ret = max8998_write_reg(i2c, reg, selector); @@ -303,14 +305,13 @@ buck1_exit: dev_dbg(max8998->dev, "BUCK2, selector:%d buck2_vol1:%d, buck2_vol2:%d\n", selector, max8998->buck2_vol[0], max8998->buck2_vol[1]); - if (gpio_is_valid(pdata->buck2_set3)) { - + if (max8998->buck2_gpio) { /* check if requested voltage */ /* value is already defined */ for (j = 0; j < ARRAY_SIZE(max8998->buck2_vol); j++) { if (max8998->buck2_vol[j] == selector) { max8998->buck2_idx = j; - buck2_gpio_set(pdata->buck2_set3, j); + buck2_gpio_set(max8998->buck2_gpio, j); goto buck2_exit; } } @@ -322,10 +323,10 @@ buck1_exit: ®, &shift, &mask); ret = max8998_write_reg(i2c, reg, selector); max8998->buck2_vol[max8998->buck2_idx] = selector; - buck2_gpio_set(pdata->buck2_set3, max8998->buck2_idx); + buck2_gpio_set(max8998->buck2_gpio, max8998->buck2_idx); buck2_exit: dev_dbg(max8998->dev, "%s: SET3:%d\n", i2c->name, - gpio_get_value(pdata->buck2_set3)); + gpiod_get_value(max8998->buck2_gpio)); } else { ret = max8998_write_reg(i2c, reg, selector); } @@ -539,36 +540,6 @@ static const struct regulator_desc regulators[] = { charger_current_table, MAX8998_REG_CHGR1, 0x7), }; -static int max8998_pmic_dt_parse_dvs_gpio(struct max8998_dev *iodev, - struct max8998_platform_data *pdata, - struct device_node *pmic_np) -{ - int gpio; - - gpio = of_get_named_gpio(pmic_np, "max8998,pmic-buck1-dvs-gpios", 0); - if (!gpio_is_valid(gpio)) { - dev_err(iodev->dev, "invalid buck1 gpio[0]: %d\n", gpio); - return -EINVAL; - } - pdata->buck1_set1 = gpio; - - gpio = of_get_named_gpio(pmic_np, "max8998,pmic-buck1-dvs-gpios", 1); - if (!gpio_is_valid(gpio)) { - dev_err(iodev->dev, "invalid buck1 gpio[1]: %d\n", gpio); - return -EINVAL; - } - pdata->buck1_set2 = gpio; - - gpio = of_get_named_gpio(pmic_np, "max8998,pmic-buck2-dvs-gpio", 0); - if (!gpio_is_valid(gpio)) { - dev_err(iodev->dev, "invalid buck 2 gpio: %d\n", gpio); - return -EINVAL; - } - pdata->buck2_set3 = gpio; - - return 0; -} - static int max8998_pmic_dt_parse_pdata(struct max8998_dev *iodev, struct max8998_platform_data *pdata) { @@ -614,10 +585,6 @@ static int max8998_pmic_dt_parse_pdata(struct max8998_dev *iodev, of_node_put(reg_np); of_node_put(regulators_np); - ret = max8998_pmic_dt_parse_dvs_gpio(iodev, pdata, pmic_np); - if (ret) - return -EINVAL; - pdata->buck_voltage_lock = of_property_read_bool(pmic_np, "max8998,pmic-buck-voltage-lock"); ret = of_property_read_u32(pmic_np, @@ -665,6 +632,7 @@ static int max8998_pmic_probe(struct platform_device *pdev) struct regulator_dev *rdev; struct max8998_data *max8998; struct i2c_client *i2c; + enum gpiod_flags flags; int i, ret; unsigned int v; @@ -693,37 +661,38 @@ static int max8998_pmic_probe(struct platform_device *pdev) max8998->buck1_idx = pdata->buck1_default_idx; max8998->buck2_idx = pdata->buck2_default_idx; - /* NOTE: */ - /* For unused GPIO NOT marked as -1 (thereof equal to 0) WARN_ON */ - /* will be displayed */ - /* Check if MAX8998 voltage selection GPIOs are defined */ - if (gpio_is_valid(pdata->buck1_set1) && - gpio_is_valid(pdata->buck1_set2)) { - /* Check if SET1 is not equal to 0 */ - if (!pdata->buck1_set1) { - dev_err(&pdev->dev, - "MAX8998 SET1 GPIO defined as 0 !\n"); - WARN_ON(!pdata->buck1_set1); - return -EIO; - } - /* Check if SET2 is not equal to 0 */ - if (!pdata->buck1_set2) { - dev_err(&pdev->dev, - "MAX8998 SET2 GPIO defined as 0 !\n"); - WARN_ON(!pdata->buck1_set2); - return -EIO; - } + flags = (max8998->buck1_idx & BIT(0)) ? GPIOD_OUT_HIGH : GPIOD_OUT_LOW; + max8998->buck1_gpio1 = devm_gpiod_get_index_optional(iodev->dev, + "max8998,pmic-buck1-dvs", + 0, + flags); + if (IS_ERR(max8998->buck1_gpio1)) + return dev_err_probe(&pdev->dev, PTR_ERR(max8998->buck1_gpio1), + "could not get BUCK1 GPIO1\n"); + gpiod_set_consumer_name(max8998->buck1_gpio1, "MAX8998 BUCK1_SET1"); - gpio_request(pdata->buck1_set1, "MAX8998 BUCK1_SET1"); - gpio_direction_output(pdata->buck1_set1, - max8998->buck1_idx & 0x1); + flags = (max8998->buck1_idx & BIT(1)) ? GPIOD_OUT_HIGH : GPIOD_OUT_LOW; + max8998->buck1_gpio2 = devm_gpiod_get_index_optional(iodev->dev, + "max8998,pmic-buck1-dvs", + 1, + flags); + if (IS_ERR(max8998->buck1_gpio2)) + return dev_err_probe(&pdev->dev, PTR_ERR(max8998->buck1_gpio2), + "could not get BUCK1 GPIO2\n"); + gpiod_set_consumer_name(max8998->buck1_gpio1, "MAX8998 BUCK1_SET2"); + flags = (max8998->buck2_idx & BIT(0)) ? GPIOD_OUT_HIGH : GPIOD_OUT_LOW; + max8998->buck2_gpio = devm_gpiod_get_index_optional(iodev->dev, + "max8998,pmic-buck2-dvs", + 0, + flags); + if (IS_ERR(max8998->buck2_gpio)) + return dev_err_probe(&pdev->dev, PTR_ERR(max8998->buck2_gpio), + "could not get BUCK2 GPIO\n"); + gpiod_set_consumer_name(max8998->buck1_gpio1, "MAX8998 BUCK2_SET3"); - gpio_request(pdata->buck1_set2, "MAX8998 BUCK1_SET2"); - gpio_direction_output(pdata->buck1_set2, - (max8998->buck1_idx >> 1) & 0x1); - + if (max8998->buck1_gpio1 && max8998->buck1_gpio2) { /* Set predefined values for BUCK1 registers */ for (v = 0; v < ARRAY_SIZE(pdata->buck1_voltage); ++v) { int index = MAX8998_BUCK1 - MAX8998_LDO2; @@ -742,18 +711,7 @@ static int max8998_pmic_probe(struct platform_device *pdev) } } - if (gpio_is_valid(pdata->buck2_set3)) { - /* Check if SET3 is not equal to 0 */ - if (!pdata->buck2_set3) { - dev_err(&pdev->dev, - "MAX8998 SET3 GPIO defined as 0 !\n"); - WARN_ON(!pdata->buck2_set3); - return -EIO; - } - gpio_request(pdata->buck2_set3, "MAX8998 BUCK2_SET3"); - gpio_direction_output(pdata->buck2_set3, - max8998->buck2_idx & 0x1); - + if (max8998->buck2_gpio) { /* Set predefined values for BUCK2 registers */ for (v = 0; v < ARRAY_SIZE(pdata->buck2_voltage); ++v) { int index = MAX8998_BUCK2 - MAX8998_LDO2; diff --git a/drivers/regulator/mp8859.c b/drivers/regulator/mp8859.c index b820bd6043e5..ab105ffd6a2e 100644 --- a/drivers/regulator/mp8859.c +++ b/drivers/regulator/mp8859.c @@ -35,6 +35,16 @@ #define MP8859_GO_BIT 0x01 +#define MP8859_IOUT_LIM_MASK 0x7f + +#define MP8859_ENABLE_MASK 0x80 +#define MP8859_DISCHG_EN_MASK 0x10 +#define MP8859_MODE_MASK 0x08 + +#define MP8859_PG_MASK 0x80 +#define MP8859_OTP_MASK 0x40 +#define MP8859_OTW_MASK 0x20 +#define MP8859_CC_CV_MASK 0x10 static int mp8859_set_voltage_sel(struct regulator_dev *rdev, unsigned int sel) { @@ -73,21 +83,221 @@ static int mp8859_get_voltage_sel(struct regulator_dev *rdev) return val; } +static int mp8859_set_voltage_time_sel(struct regulator_dev *rdev, + unsigned int from, unsigned int to) +{ + int change; + + /* The voltage ramps at 1mV/uS, selectors are 10mV */ + if (from > to) + change = from - to; + else + change = to - from; + + return change * 10 * 1000; +} + +static unsigned int mp8859_get_mode(struct regulator_dev *rdev) +{ + unsigned int val; + int ret; + + ret = regmap_read(rdev->regmap, MP8859_CTL1_REG, &val); + if (ret != 0) { + dev_err(&rdev->dev, "Failed to read mode: %d\n", ret); + return 0; + } + + if (val & MP8859_MODE_MASK) + return REGULATOR_MODE_FAST; + else + return REGULATOR_MODE_NORMAL; +} + +static int mp8859_set_mode(struct regulator_dev *rdev, unsigned int mode) +{ + unsigned int val; + + switch (mode) { + case REGULATOR_MODE_FAST: + val = MP8859_MODE_MASK; + break; + case REGULATOR_MODE_NORMAL: + val = 0; + break; + default: + return -EINVAL; + } + + return regmap_update_bits(rdev->regmap, MP8859_CTL1_REG, + MP8859_MODE_MASK, val); +} + +static int mp8859_set_current_limit(struct regulator_dev *rdev, + int min_uA, int max_uA) +{ + unsigned int cur_val, new_val; + int ret, i; + + /* Steps of 50mA */ + new_val = max_uA / 50000; + if (new_val > MP8859_IOUT_LIM_MASK) + return -EINVAL; + if (new_val == 0) + return -EINVAL; + + /* + * If the regulator is limiting then ramp gradually as per + * datasheet, otherwise just set the value directly. + */ + ret = regmap_read(rdev->regmap, MP8859_STATUS_REG, &cur_val); + if (ret != 0) + return ret; + if (!(cur_val & MP8859_CC_CV_MASK)) { + return regmap_update_bits(rdev->regmap, MP8859_IOUT_LIM_REG, + MP8859_IOUT_LIM_MASK, new_val); + } + + ret = regmap_read(rdev->regmap, MP8859_IOUT_LIM_REG, &cur_val); + if (ret != 0) + return ret; + + if (cur_val >= new_val) { + for (i = cur_val; i >= new_val; i--) { + ret = regmap_update_bits(rdev->regmap, + MP8859_IOUT_LIM_REG, + MP8859_IOUT_LIM_MASK, + cur_val - i); + if (ret != 0) + return ret; + } + } else { + for (i = cur_val; i <= new_val; i++) { + ret = regmap_update_bits(rdev->regmap, + MP8859_IOUT_LIM_REG, + MP8859_IOUT_LIM_MASK, + cur_val + i); + if (ret != 0) + return ret; + } + } + + return 0; +} + +static int mp8859_get_status(struct regulator_dev *rdev) +{ + unsigned int val; + int ret; + + /* Output status is only meaingful when enabled */ + ret = regmap_read(rdev->regmap, MP8859_CTL1_REG, &val); + if (ret != 0) + return ret; + if (!(val & MP8859_ENABLE_MASK)) + return REGULATOR_STATUS_UNDEFINED; + + ret = regmap_read(rdev->regmap, MP8859_STATUS_REG, &val); + if (ret != 0) + return ret; + + if (val & MP8859_PG_MASK) + return REGULATOR_STATUS_ON; + else + return REGULATOR_STATUS_ERROR; +} + +static int mp8859_get_error_flags(struct regulator_dev *rdev, + unsigned int *flags) +{ + unsigned int status, enabled; + int ret; + + *flags = 0; + + /* Output status is only meaingful when enabled */ + ret = regmap_read(rdev->regmap, MP8859_CTL1_REG, &enabled); + if (ret != 0) + return ret; + enabled &= MP8859_ENABLE_MASK; + + ret = regmap_read(rdev->regmap, MP8859_STATUS_REG, &status); + if (ret != 0) + return ret; + + if (enabled && !(status & MP8859_PG_MASK)) + status |= REGULATOR_ERROR_FAIL; + if (status & MP8859_OTP_MASK) + status |= REGULATOR_ERROR_OVER_TEMP; + if (status & MP8859_OTW_MASK) + status |= REGULATOR_ERROR_OVER_TEMP_WARN; + if (status & MP8859_CC_CV_MASK) + status |= REGULATOR_ERROR_OVER_CURRENT; + + return 0; +} + static const struct linear_range mp8859_dcdc_ranges[] = { REGULATOR_LINEAR_RANGE(0, VOL_MIN_IDX, VOL_MAX_IDX, 10000), }; +static bool mp8859_readable(struct device *dev, unsigned int reg) +{ + switch (reg) { + case MP8859_VOUT_L_REG: + case MP8859_VOUT_H_REG: + case MP8859_VOUT_GO_REG: + case MP8859_IOUT_LIM_REG: + case MP8859_CTL1_REG: + case MP8859_CTL2_REG: + case MP8859_STATUS_REG: + case MP8859_INTERRUPT_REG: + case MP8859_MASK_REG: + case MP8859_ID1_REG: + case MP8859_MFR_ID_REG: + case MP8859_DEV_ID_REG: + case MP8859_IC_REV_REG: + return true; + default: + return false; + } +} + +static bool mp8859_volatile(struct device *dev, unsigned int reg) +{ + switch (reg) { + case MP8859_VOUT_GO_REG: + case MP8859_STATUS_REG: + case MP8859_INTERRUPT_REG: + return true; + default: + return false; + } +} + static const struct regmap_config mp8859_regmap = { .reg_bits = 8, .val_bits = 8, .max_register = MP8859_MAX_REG, - .cache_type = REGCACHE_RBTREE, + .cache_type = REGCACHE_MAPLE, + .readable_reg = mp8859_readable, + .volatile_reg = mp8859_volatile, }; static const struct regulator_ops mp8859_ops = { .set_voltage_sel = mp8859_set_voltage_sel, .get_voltage_sel = mp8859_get_voltage_sel, .list_voltage = regulator_list_voltage_linear_range, + .set_voltage_time_sel = mp8859_set_voltage_time_sel, + .enable = regulator_enable_regmap, + .disable = regulator_disable_regmap, + .is_enabled = regulator_is_enabled_regmap, + .set_mode = mp8859_set_mode, + .get_mode = mp8859_get_mode, + .set_active_discharge = regulator_set_active_discharge_regmap, + .set_current_limit = mp8859_set_current_limit, + .get_status = mp8859_get_status, + .get_error_flags = mp8859_get_error_flags, }; static const struct regulator_desc mp8859_regulators[] = { @@ -100,6 +310,12 @@ static const struct regulator_desc mp8859_regulators[] = { .n_voltages = VOL_MAX_IDX + 1, .linear_ranges = mp8859_dcdc_ranges, .n_linear_ranges = 1, + .enable_reg = MP8859_CTL1_REG, + .enable_mask = MP8859_ENABLE_MASK, + .enable_val = MP8859_ENABLE_MASK, + .active_discharge_reg = MP8859_CTL1_REG, + .active_discharge_on = MP8859_DISCHG_EN_MASK, + .active_discharge_mask = MP8859_DISCHG_EN_MASK, .ops = &mp8859_ops, .owner = THIS_MODULE, }, @@ -111,12 +327,46 @@ static int mp8859_i2c_probe(struct i2c_client *i2c) struct regulator_config config = {.dev = &i2c->dev}; struct regmap *regmap = devm_regmap_init_i2c(i2c, &mp8859_regmap); struct regulator_dev *rdev; + unsigned int val, rev; if (IS_ERR(regmap)) { ret = PTR_ERR(regmap); dev_err(&i2c->dev, "regmap init failed: %d\n", ret); return ret; } + + ret = regmap_read(regmap, MP8859_MFR_ID_REG, &val); + if (ret != 0) { + dev_err(&i2c->dev, "Failed to read manufacturer ID: %d\n", ret); + return ret; + } + if (val != 0x9) { + dev_err(&i2c->dev, "Manufacturer ID %x != 9\n", val); + return -EINVAL; + } + + ret = regmap_read(regmap, MP8859_DEV_ID_REG, &val); + if (ret != 0) { + dev_err(&i2c->dev, "Failed to read device ID: %d\n", ret); + return ret; + } + if (val != 0x58) { + dev_err(&i2c->dev, "Manufacturer ID %x != 0x58\n", val); + return -EINVAL; + } + + ret = regmap_read(regmap, MP8859_IC_REV_REG, &rev); + if (ret != 0) { + dev_err(&i2c->dev, "Failed to read device revision: %d\n", ret); + return ret; + } + ret = regmap_read(regmap, MP8859_ID1_REG, &val); + if (ret != 0) { + dev_err(&i2c->dev, "Failed to read device ID1: %d\n", ret); + return ret; + } + dev_info(&i2c->dev, "MP8859-%04d revision %d\n", val, rev); + rdev = devm_regulator_register(&i2c->dev, &mp8859_regulators[0], &config); diff --git a/drivers/regulator/pwm-regulator.c b/drivers/regulator/pwm-regulator.c index 60cfcd741c2a..7434b6b22d32 100644 --- a/drivers/regulator/pwm-regulator.c +++ b/drivers/regulator/pwm-regulator.c @@ -271,11 +271,10 @@ static int pwm_regulator_init_table(struct platform_device *pdev, of_find_property(np, "voltage-table", &length); if ((length < sizeof(*duty_cycle_table)) || - (length % sizeof(*duty_cycle_table))) { - dev_err(&pdev->dev, "voltage-table length(%d) is invalid\n", - length); - return -EINVAL; - } + (length % sizeof(*duty_cycle_table))) + return dev_err_probe(&pdev->dev, -EINVAL, + "voltage-table length(%d) is invalid\n", + length); duty_cycle_table = devm_kzalloc(&pdev->dev, length, GFP_KERNEL); if (!duty_cycle_table) @@ -284,10 +283,9 @@ static int pwm_regulator_init_table(struct platform_device *pdev, ret = of_property_read_u32_array(np, "voltage-table", (u32 *)duty_cycle_table, length / sizeof(u32)); - if (ret) { - dev_err(&pdev->dev, "Failed to read voltage-table: %d\n", ret); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Failed to read voltage-table\n"); drvdata->state = -ENOTRECOVERABLE; drvdata->duty_cycle_table = duty_cycle_table; @@ -359,10 +357,9 @@ static int pwm_regulator_probe(struct platform_device *pdev) enum gpiod_flags gpio_flags; int ret; - if (!np) { - dev_err(&pdev->dev, "Device Tree node missing\n"); - return -EINVAL; - } + if (!np) + return dev_err_probe(&pdev->dev, -EINVAL, + "Device Tree node missing\n"); drvdata = devm_kzalloc(&pdev->dev, sizeof(*drvdata), GFP_KERNEL); if (!drvdata) @@ -400,8 +397,7 @@ static int pwm_regulator_probe(struct platform_device *pdev) gpio_flags); if (IS_ERR(drvdata->enb_gpio)) { ret = PTR_ERR(drvdata->enb_gpio); - dev_err(&pdev->dev, "Failed to get enable GPIO: %d\n", ret); - return ret; + return dev_err_probe(&pdev->dev, ret, "Failed to get enable GPIO\n"); } ret = pwm_adjust_config(drvdata->pwm); @@ -409,19 +405,17 @@ static int pwm_regulator_probe(struct platform_device *pdev) return ret; ret = pwm_regulator_init_boot_on(pdev, drvdata, init_data); - if (ret) { - dev_err(&pdev->dev, "Failed to apply boot_on settings: %d\n", - ret); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Failed to apply boot_on settings\n"); regulator = devm_regulator_register(&pdev->dev, &drvdata->desc, &config); if (IS_ERR(regulator)) { ret = PTR_ERR(regulator); - dev_err(&pdev->dev, "Failed to register regulator %s: %d\n", - drvdata->desc.name, ret); - return ret; + return dev_err_probe(&pdev->dev, ret, + "Failed to register regulator %s\n", + drvdata->desc.name); } return 0; diff --git a/drivers/regulator/qcom_smd-regulator.c b/drivers/regulator/qcom_smd-regulator.c index d1be9568025e..3b7e06b9f5ce 100644 --- a/drivers/regulator/qcom_smd-regulator.c +++ b/drivers/regulator/qcom_smd-regulator.c @@ -11,11 +11,10 @@ #include #include +struct qcom_smd_rpm *smd_vreg_rpm; + struct qcom_rpm_reg { struct device *dev; - - struct qcom_smd_rpm *rpm; - u32 type; u32 id; @@ -70,7 +69,7 @@ static int rpm_reg_write_active(struct qcom_rpm_reg *vreg) if (!reqlen) return 0; - ret = qcom_rpm_smd_write(vreg->rpm, QCOM_SMD_RPM_ACTIVE_STATE, + ret = qcom_rpm_smd_write(smd_vreg_rpm, QCOM_SMD_RPM_ACTIVE_STATE, vreg->type, vreg->id, req, sizeof(req[0]) * reqlen); if (!ret) { @@ -1384,14 +1383,13 @@ MODULE_DEVICE_TABLE(of, rpm_of_match); * @dev: Pointer to the top level qcom_smd-regulator PMIC device * @node: Pointer to the individual qcom_smd-regulator resource * device node - * @rpm: Pointer to the rpm bus node * @pmic_rpm_data: Pointer to a null-terminated array of qcom_smd-regulator * resources defined for the top level PMIC device * * Return: 0 on success, errno on failure */ static int rpm_regulator_init_vreg(struct qcom_rpm_reg *vreg, struct device *dev, - struct device_node *node, struct qcom_smd_rpm *rpm, + struct device_node *node, const struct rpm_regulator_data *pmic_rpm_data) { struct regulator_config config = {}; @@ -1409,7 +1407,6 @@ static int rpm_regulator_init_vreg(struct qcom_rpm_reg *vreg, struct device *dev } vreg->dev = dev; - vreg->rpm = rpm; vreg->type = rpm_data->type; vreg->id = rpm_data->id; @@ -1449,6 +1446,11 @@ static int rpm_reg_probe(struct platform_device *pdev) return -ENODEV; } + if (smd_vreg_rpm && rpm != smd_vreg_rpm) + return dev_err_probe(dev, -EINVAL, "RPM mismatch\n"); + + smd_vreg_rpm = rpm; + vreg_data = of_device_get_match_data(dev); if (!vreg_data) return -ENODEV; @@ -1460,8 +1462,7 @@ static int rpm_reg_probe(struct platform_device *pdev) return -ENOMEM; } - ret = rpm_regulator_init_vreg(vreg, dev, node, rpm, vreg_data); - + ret = rpm_regulator_init_vreg(vreg, dev, node, vreg_data); if (ret < 0) { of_node_put(node); return ret; diff --git a/drivers/regulator/userspace-consumer.c b/drivers/regulator/userspace-consumer.c index 53d1b9d6f69c..86a626a4f610 100644 --- a/drivers/regulator/userspace-consumer.c +++ b/drivers/regulator/userspace-consumer.c @@ -208,6 +208,7 @@ static const struct of_device_id regulator_userspace_consumer_of_match[] = { { .compatible = "regulator-output", }, {}, }; +MODULE_DEVICE_TABLE(of, regulator_userspace_consumer_of_match); static struct platform_driver regulator_userspace_consumer_driver = { .probe = regulator_userspace_consumer_probe, diff --git a/drivers/remoteproc/imx_dsp_rproc.c b/drivers/remoteproc/imx_dsp_rproc.c index a1c62d15f16c..d73727a5828a 100644 --- a/drivers/remoteproc/imx_dsp_rproc.c +++ b/drivers/remoteproc/imx_dsp_rproc.c @@ -103,12 +103,10 @@ enum imx_dsp_rp_mbox_messages { * @tx_ch: mailbox tx channel handle * @rx_ch: mailbox rx channel handle * @rxdb_ch: mailbox rx doorbell channel handle - * @pd_dev: power domain device - * @pd_dev_link: power domain device link + * @pd_list: power domain list * @ipc_handle: System Control Unit ipc handle * @rproc_work: work for processing virtio interrupts * @pm_comp: completion primitive to sync for suspend response - * @num_domains: power domain number * @flags: control flags */ struct imx_dsp_rproc { @@ -121,12 +119,10 @@ struct imx_dsp_rproc { struct mbox_chan *tx_ch; struct mbox_chan *rx_ch; struct mbox_chan *rxdb_ch; - struct device **pd_dev; - struct device_link **pd_dev_link; + struct dev_pm_domain_list *pd_list; struct imx_sc_ipc *ipc_handle; struct work_struct rproc_work; struct completion pm_comp; - int num_domains; u32 flags; }; @@ -955,74 +951,14 @@ static const struct rproc_ops imx_dsp_rproc_ops = { static int imx_dsp_attach_pm_domains(struct imx_dsp_rproc *priv) { struct device *dev = priv->rproc->dev.parent; - int ret, i; + int ret; - priv->num_domains = of_count_phandle_with_args(dev->of_node, - "power-domains", - "#power-domain-cells"); - - /* If only one domain, then no need to link the device */ - if (priv->num_domains <= 1) + /* A single PM domain is already attached. */ + if (dev->pm_domain) return 0; - priv->pd_dev = devm_kmalloc_array(dev, priv->num_domains, - sizeof(*priv->pd_dev), - GFP_KERNEL); - if (!priv->pd_dev) - return -ENOMEM; - - priv->pd_dev_link = devm_kmalloc_array(dev, priv->num_domains, - sizeof(*priv->pd_dev_link), - GFP_KERNEL); - if (!priv->pd_dev_link) - return -ENOMEM; - - for (i = 0; i < priv->num_domains; i++) { - priv->pd_dev[i] = dev_pm_domain_attach_by_id(dev, i); - if (IS_ERR(priv->pd_dev[i])) { - ret = PTR_ERR(priv->pd_dev[i]); - goto detach_pm; - } - - /* - * device_link_add will check priv->pd_dev[i], if it is - * NULL, then will break. - */ - priv->pd_dev_link[i] = device_link_add(dev, - priv->pd_dev[i], - DL_FLAG_STATELESS | - DL_FLAG_PM_RUNTIME); - if (!priv->pd_dev_link[i]) { - dev_pm_domain_detach(priv->pd_dev[i], false); - ret = -EINVAL; - goto detach_pm; - } - } - - return 0; - -detach_pm: - while (--i >= 0) { - device_link_del(priv->pd_dev_link[i]); - dev_pm_domain_detach(priv->pd_dev[i], false); - } - - return ret; -} - -static int imx_dsp_detach_pm_domains(struct imx_dsp_rproc *priv) -{ - int i; - - if (priv->num_domains <= 1) - return 0; - - for (i = 0; i < priv->num_domains; i++) { - device_link_del(priv->pd_dev_link[i]); - dev_pm_domain_detach(priv->pd_dev[i], false); - } - - return 0; + ret = dev_pm_domain_attach_list(dev, NULL, &priv->pd_list); + return ret < 0 ? ret : 0; } /** @@ -1154,7 +1090,7 @@ static int imx_dsp_rproc_probe(struct platform_device *pdev) return 0; err_detach_domains: - imx_dsp_detach_pm_domains(priv); + dev_pm_domain_detach_list(priv->pd_list); err_put_rproc: rproc_free(rproc); @@ -1168,7 +1104,7 @@ static void imx_dsp_rproc_remove(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); rproc_del(rproc); - imx_dsp_detach_pm_domains(priv); + dev_pm_domain_detach_list(priv->pd_list); rproc_free(rproc); } diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c index 8bb293b9f327..3161f14442bc 100644 --- a/drivers/remoteproc/imx_rproc.c +++ b/drivers/remoteproc/imx_rproc.c @@ -92,7 +92,6 @@ struct imx_rproc_mem { static int imx_rproc_xtr_mbox_init(struct rproc *rproc); static void imx_rproc_free_mbox(struct rproc *rproc); -static int imx_rproc_detach_pd(struct rproc *rproc); struct imx_rproc { struct device *dev; @@ -113,10 +112,8 @@ struct imx_rproc { u32 rproc_pt; /* partition id */ u32 rsrc_id; /* resource id */ u32 entry; /* cpu start address */ - int num_pd; u32 core_index; - struct device **pd_dev; - struct device_link **pd_dev_link; + struct dev_pm_domain_list *pd_list; }; static const struct imx_rproc_att imx_rproc_att_imx93[] = { @@ -853,7 +850,7 @@ static void imx_rproc_put_scu(struct rproc *rproc) return; if (imx_sc_rm_is_resource_owned(priv->ipc_handle, priv->rsrc_id)) { - imx_rproc_detach_pd(rproc); + dev_pm_domain_detach_list(priv->pd_list); return; } @@ -880,72 +877,20 @@ static int imx_rproc_partition_notify(struct notifier_block *nb, static int imx_rproc_attach_pd(struct imx_rproc *priv) { struct device *dev = priv->dev; - int ret, i; + int ret; + struct dev_pm_domain_attach_data pd_data = { + .pd_flags = PD_FLAG_DEV_LINK_ON, + }; /* * If there is only one power-domain entry, the platform driver framework * will handle it, no need handle it in this driver. */ - priv->num_pd = of_count_phandle_with_args(dev->of_node, "power-domains", - "#power-domain-cells"); - if (priv->num_pd <= 1) + if (dev->pm_domain) return 0; - priv->pd_dev = devm_kmalloc_array(dev, priv->num_pd, sizeof(*priv->pd_dev), GFP_KERNEL); - if (!priv->pd_dev) - return -ENOMEM; - - priv->pd_dev_link = devm_kmalloc_array(dev, priv->num_pd, sizeof(*priv->pd_dev_link), - GFP_KERNEL); - - if (!priv->pd_dev_link) - return -ENOMEM; - - for (i = 0; i < priv->num_pd; i++) { - priv->pd_dev[i] = dev_pm_domain_attach_by_id(dev, i); - if (IS_ERR(priv->pd_dev[i])) { - ret = PTR_ERR(priv->pd_dev[i]); - goto detach_pd; - } - - priv->pd_dev_link[i] = device_link_add(dev, priv->pd_dev[i], DL_FLAG_STATELESS | - DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE); - if (!priv->pd_dev_link[i]) { - dev_pm_domain_detach(priv->pd_dev[i], false); - ret = -EINVAL; - goto detach_pd; - } - } - - return 0; - -detach_pd: - while (--i >= 0) { - device_link_del(priv->pd_dev_link[i]); - dev_pm_domain_detach(priv->pd_dev[i], false); - } - - return ret; -} - -static int imx_rproc_detach_pd(struct rproc *rproc) -{ - struct imx_rproc *priv = rproc->priv; - int i; - - /* - * If there is only one power-domain entry, the platform driver framework - * will handle it, no need handle it in this driver. - */ - if (priv->num_pd <= 1) - return 0; - - for (i = 0; i < priv->num_pd; i++) { - device_link_del(priv->pd_dev_link[i]); - dev_pm_domain_detach(priv->pd_dev[i], false); - } - - return 0; + ret = dev_pm_domain_attach_list(dev, &pd_data, &priv->pd_list); + return ret < 0 ? ret : 0; } static int imx_rproc_detect_mode(struct imx_rproc *priv) diff --git a/drivers/remoteproc/qcom_q6v5_adsp.c b/drivers/remoteproc/qcom_q6v5_adsp.c index 6c67514cc493..93f9a1537ec6 100644 --- a/drivers/remoteproc/qcom_q6v5_adsp.c +++ b/drivers/remoteproc/qcom_q6v5_adsp.c @@ -55,8 +55,6 @@ #define QDSP6SS_CORE_CBCR 0x20 #define QDSP6SS_SLEEP_CBCR 0x3c -#define QCOM_Q6V5_RPROC_PROXY_PD_MAX 3 - #define LPASS_BOOT_CORE_START BIT(0) #define LPASS_BOOT_CMD_START BIT(0) #define LPASS_EFUSE_Q6SS_EVB_SEL 0x0 @@ -74,7 +72,8 @@ struct adsp_pil_data { const char **clk_ids; int num_clks; - const char **proxy_pd_names; + const char **pd_names; + unsigned int num_pds; const char *load_state; }; @@ -110,8 +109,7 @@ struct qcom_adsp { size_t mem_size; bool has_iommu; - struct device *proxy_pds[QCOM_Q6V5_RPROC_PROXY_PD_MAX]; - size_t proxy_pd_count; + struct dev_pm_domain_list *pd_list; struct qcom_rproc_glink glink_subdev; struct qcom_rproc_ssr ssr_subdev; @@ -120,98 +118,92 @@ struct qcom_adsp { int (*shutdown)(struct qcom_adsp *adsp); }; -static int qcom_rproc_pds_attach(struct device *dev, struct qcom_adsp *adsp, - const char **pd_names) +static int qcom_rproc_pds_attach(struct qcom_adsp *adsp, const char **pd_names, + unsigned int num_pds) { - struct device **devs = adsp->proxy_pds; - size_t num_pds = 0; + struct device *dev = adsp->dev; + struct dev_pm_domain_attach_data pd_data = { + .pd_names = pd_names, + .num_pd_names = num_pds, + }; int ret; - int i; + + /* Handle single power domain */ + if (dev->pm_domain) + goto out; if (!pd_names) return 0; - /* Handle single power domain */ - if (dev->pm_domain) { - devs[0] = dev; - pm_runtime_enable(dev); - return 1; - } + ret = dev_pm_domain_attach_list(dev, &pd_data, &adsp->pd_list); + if (ret < 0) + return ret; - while (pd_names[num_pds]) - num_pds++; - - if (num_pds > ARRAY_SIZE(adsp->proxy_pds)) - return -E2BIG; - - for (i = 0; i < num_pds; i++) { - devs[i] = dev_pm_domain_attach_by_name(dev, pd_names[i]); - if (IS_ERR_OR_NULL(devs[i])) { - ret = PTR_ERR(devs[i]) ? : -ENODATA; - goto unroll_attach; - } - } - - return num_pds; - -unroll_attach: - for (i--; i >= 0; i--) - dev_pm_domain_detach(devs[i], false); - - return ret; +out: + pm_runtime_enable(dev); + return 0; } -static void qcom_rproc_pds_detach(struct qcom_adsp *adsp, struct device **pds, - size_t pd_count) +static void qcom_rproc_pds_detach(struct qcom_adsp *adsp) { struct device *dev = adsp->dev; - int i; + struct dev_pm_domain_list *pds = adsp->pd_list; - /* Handle single power domain */ - if (dev->pm_domain && pd_count) { - pm_runtime_disable(dev); - return; - } + dev_pm_domain_detach_list(pds); - for (i = 0; i < pd_count; i++) - dev_pm_domain_detach(pds[i], false); + if (dev->pm_domain || pds) + pm_runtime_disable(adsp->dev); } -static int qcom_rproc_pds_enable(struct qcom_adsp *adsp, struct device **pds, - size_t pd_count) +static int qcom_rproc_pds_enable(struct qcom_adsp *adsp) { - int ret; - int i; + struct device *dev = adsp->dev; + struct dev_pm_domain_list *pds = adsp->pd_list; + int ret, i = 0; - for (i = 0; i < pd_count; i++) { - dev_pm_genpd_set_performance_state(pds[i], INT_MAX); - ret = pm_runtime_resume_and_get(pds[i]); - if (ret < 0) { - dev_pm_genpd_set_performance_state(pds[i], 0); - goto unroll_pd_votes; - } + if (!dev->pm_domain && !pds) + return 0; + + if (dev->pm_domain) + dev_pm_genpd_set_performance_state(dev, INT_MAX); + + while (pds && i < pds->num_pds) { + dev_pm_genpd_set_performance_state(pds->pd_devs[i], INT_MAX); + i++; } - return 0; + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + while (pds && i > 0) { + i--; + dev_pm_genpd_set_performance_state(pds->pd_devs[i], 0); + } -unroll_pd_votes: - for (i--; i >= 0; i--) { - dev_pm_genpd_set_performance_state(pds[i], 0); - pm_runtime_put(pds[i]); + if (dev->pm_domain) + dev_pm_genpd_set_performance_state(dev, 0); } return ret; } -static void qcom_rproc_pds_disable(struct qcom_adsp *adsp, struct device **pds, - size_t pd_count) +static void qcom_rproc_pds_disable(struct qcom_adsp *adsp) { - int i; + struct device *dev = adsp->dev; + struct dev_pm_domain_list *pds = adsp->pd_list; + int i = 0; - for (i = 0; i < pd_count; i++) { - dev_pm_genpd_set_performance_state(pds[i], 0); - pm_runtime_put(pds[i]); + if (!dev->pm_domain && !pds) + return; + + if (dev->pm_domain) + dev_pm_genpd_set_performance_state(dev, 0); + + while (pds && i < pds->num_pds) { + dev_pm_genpd_set_performance_state(pds->pd_devs[i], 0); + i++; } + + pm_runtime_put(dev); } static int qcom_wpss_shutdown(struct qcom_adsp *adsp) @@ -397,8 +389,7 @@ static int adsp_start(struct rproc *rproc) if (ret) goto adsp_smmu_unmap; - ret = qcom_rproc_pds_enable(adsp, adsp->proxy_pds, - adsp->proxy_pd_count); + ret = qcom_rproc_pds_enable(adsp); if (ret < 0) goto disable_xo_clk; @@ -448,7 +439,7 @@ static int adsp_start(struct rproc *rproc) disable_adsp_clks: clk_bulk_disable_unprepare(adsp->num_clks, adsp->clks); disable_power_domain: - qcom_rproc_pds_disable(adsp, adsp->proxy_pds, adsp->proxy_pd_count); + qcom_rproc_pds_disable(adsp); disable_xo_clk: clk_disable_unprepare(adsp->xo); adsp_smmu_unmap: @@ -464,7 +455,7 @@ static void qcom_adsp_pil_handover(struct qcom_q6v5 *q6v5) struct qcom_adsp *adsp = container_of(q6v5, struct qcom_adsp, q6v5); clk_disable_unprepare(adsp->xo); - qcom_rproc_pds_disable(adsp, adsp->proxy_pds, adsp->proxy_pd_count); + qcom_rproc_pds_disable(adsp); } static int adsp_stop(struct rproc *rproc) @@ -715,13 +706,11 @@ static int adsp_probe(struct platform_device *pdev) if (ret) goto free_rproc; - ret = qcom_rproc_pds_attach(adsp->dev, adsp, - desc->proxy_pd_names); + ret = qcom_rproc_pds_attach(adsp, desc->pd_names, desc->num_pds); if (ret < 0) { dev_err(&pdev->dev, "Failed to attach proxy power domains\n"); goto free_rproc; } - adsp->proxy_pd_count = ret; ret = adsp_init_reset(adsp); if (ret) @@ -753,7 +742,7 @@ static int adsp_probe(struct platform_device *pdev) return 0; disable_pm: - qcom_rproc_pds_detach(adsp, adsp->proxy_pds, adsp->proxy_pd_count); + qcom_rproc_pds_detach(adsp); free_rproc: rproc_free(rproc); @@ -771,7 +760,7 @@ static void adsp_remove(struct platform_device *pdev) qcom_remove_glink_subdev(adsp->rproc, &adsp->glink_subdev); qcom_remove_sysmon_subdev(adsp->sysmon); qcom_remove_ssr_subdev(adsp->rproc, &adsp->ssr_subdev); - qcom_rproc_pds_detach(adsp, adsp->proxy_pds, adsp->proxy_pd_count); + qcom_rproc_pds_detach(adsp); rproc_free(adsp->rproc); } @@ -788,9 +777,8 @@ static const struct adsp_pil_data adsp_resource_init = { "qdsp6ss_xo", "qdsp6ss_sleep", "qdsp6ss_core", NULL }, .num_clks = 7, - .proxy_pd_names = (const char*[]) { - "cx", NULL - }, + .pd_names = (const char*[]) { "cx" }, + .num_pds = 1, }; static const struct adsp_pil_data adsp_sc7280_resource_init = { @@ -821,9 +809,8 @@ static const struct adsp_pil_data cdsp_resource_init = { "q6_axim", NULL }, .num_clks = 7, - .proxy_pd_names = (const char*[]) { - "cx", NULL - }, + .pd_names = (const char*[]) { "cx" }, + .num_pds = 1, }; static const struct adsp_pil_data wpss_resource_init = { @@ -839,9 +826,8 @@ static const struct adsp_pil_data wpss_resource_init = { "ahb_bdg", "ahb", "rscp", NULL }, .num_clks = 3, - .proxy_pd_names = (const char*[]) { - "cx", "mx", NULL - }, + .pd_names = (const char*[]) { "cx", "mx" }, + .num_pds = 2, }; static const struct of_device_id adsp_of_match[] = { diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index ddae0fde798e..bc7021da2fe9 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -694,7 +694,7 @@ config SPI_MTK_SNFI This enables support for SPI-NAND mode on the MediaTek NAND Flash Interface found on MediaTek ARM SoCs. This controller is implemented as a SPI-MEM controller with pipelined ECC - capcability. + capability. config SPI_WPCM_FIU tristate "Nuvoton WPCM450 Flash Interface Unit" diff --git a/drivers/spi/spi-ath79.c b/drivers/spi/spi-ath79.c index b7ada981464a..d78762d4db98 100644 --- a/drivers/spi/spi-ath79.c +++ b/drivers/spi/spi-ath79.c @@ -189,7 +189,7 @@ static int ath79_spi_probe(struct platform_device *pdev) host->num_chipselect = 3; host->mem_ops = &ath79_mem_ops; - sp->bitbang.master = host; + sp->bitbang.ctlr = host; sp->bitbang.chipselect = ath79_spi_chipselect; sp->bitbang.txrx_word[SPI_MODE_0] = ath79_spi_txrx_mode0; sp->bitbang.flags = SPI_CS_HIGH; @@ -237,7 +237,7 @@ static void ath79_spi_remove(struct platform_device *pdev) spi_bitbang_stop(&sp->bitbang); ath79_spi_disable(sp); - spi_controller_put(sp->bitbang.master); + spi_controller_put(sp->bitbang.ctlr); } static void ath79_spi_shutdown(struct platform_device *pdev) diff --git a/drivers/spi/spi-au1550.c b/drivers/spi/spi-au1550.c index 1011b1a8f241..825d2f1cdff8 100644 --- a/drivers/spi/spi-au1550.c +++ b/drivers/spi/spi-au1550.c @@ -800,7 +800,7 @@ static int au1550_spi_probe(struct platform_device *pdev) init_completion(&hw->host_done); - hw->bitbang.master = hw->host; + hw->bitbang.ctlr = hw->host; hw->bitbang.setup_transfer = au1550_spi_setupxfer; hw->bitbang.chipselect = au1550_spi_chipsel; hw->bitbang.txrx_bufs = au1550_spi_txrx_bufs; diff --git a/drivers/spi/spi-axi-spi-engine.c b/drivers/spi/spi-axi-spi-engine.c index 9ace259d2d29..7cc219d78551 100644 --- a/drivers/spi/spi-axi-spi-engine.c +++ b/drivers/spi/spi-axi-spi-engine.c @@ -6,20 +6,15 @@ */ #include -#include +#include +#include #include #include #include #include +#include #include #include -#include - -#define SPI_ENGINE_VERSION_MAJOR(x) ((x >> 16) & 0xff) -#define SPI_ENGINE_VERSION_MINOR(x) ((x >> 8) & 0xff) -#define SPI_ENGINE_VERSION_PATCH(x) (x & 0xff) - -#define SPI_ENGINE_REG_VERSION 0x00 #define SPI_ENGINE_REG_RESET 0x40 @@ -62,6 +57,9 @@ #define SPI_ENGINE_TRANSFER_WRITE 0x1 #define SPI_ENGINE_TRANSFER_READ 0x2 +/* Arbitrary sync ID for use by host->cur_msg */ +#define AXI_SPI_ENGINE_CUR_MSG_SYNC_ID 0x1 + #define SPI_ENGINE_CMD(inst, arg1, arg2) \ (((inst) << 12) | ((arg1) << 8) | (arg2)) @@ -78,15 +76,13 @@ struct spi_engine_program { unsigned int length; - uint16_t instructions[]; + uint16_t instructions[] __counted_by(length); }; /** * struct spi_engine_message_state - SPI engine per-message state */ struct spi_engine_message_state { - /** @p: Instructions for executing this message. */ - struct spi_engine_program *p; /** @cmd_length: Number of elements in cmd_buf array. */ unsigned cmd_length; /** @cmd_buf: Array of commands not yet written to CMD FIFO. */ @@ -103,8 +99,6 @@ struct spi_engine_message_state { unsigned int rx_length; /** @rx_buf: Bytes not yet written to the RX FIFO. */ uint8_t *rx_buf; - /** @sync_id: ID to correlate SYNC interrupts with this message. */ - u8 sync_id; }; struct spi_engine { @@ -114,19 +108,18 @@ struct spi_engine { spinlock_t lock; void __iomem *base; - struct ida sync_ida; - struct timer_list watchdog_timer; - struct spi_controller *controller; - + struct spi_engine_message_state msg_state; + struct completion msg_complete; unsigned int int_enable; }; static void spi_engine_program_add_cmd(struct spi_engine_program *p, bool dry, uint16_t cmd) { - if (!dry) - p->instructions[p->length] = cmd; p->length++; + + if (!dry) + p->instructions[p->length - 1] = cmd; } static unsigned int spi_engine_get_config(struct spi_device *spi) @@ -488,14 +481,10 @@ static irqreturn_t spi_engine_irq(int irq, void *devid) } if (pending & SPI_ENGINE_INT_SYNC && msg) { - struct spi_engine_message_state *st = msg->state; - - if (completed_id == st->sync_id) { - if (timer_delete_sync(&spi_engine->watchdog_timer)) { - msg->status = 0; - msg->actual_length = msg->frame_length; - spi_finalize_current_message(host); - } + if (completed_id == AXI_SPI_ENGINE_CUR_MSG_SYNC_ID) { + msg->status = 0; + msg->actual_length = msg->frame_length; + complete(&spi_engine->msg_complete); disable_int |= SPI_ENGINE_INT_SYNC; } } @@ -511,61 +500,32 @@ static irqreturn_t spi_engine_irq(int irq, void *devid) return IRQ_HANDLED; } -static int spi_engine_prepare_message(struct spi_controller *host, - struct spi_message *msg) +static int spi_engine_optimize_message(struct spi_message *msg) { struct spi_engine_program p_dry, *p; - struct spi_engine *spi_engine = spi_controller_get_devdata(host); - struct spi_engine_message_state *st; - size_t size; - int ret; - - st = kzalloc(sizeof(*st), GFP_KERNEL); - if (!st) - return -ENOMEM; spi_engine_precompile_message(msg); p_dry.length = 0; spi_engine_compile_message(msg, true, &p_dry); - size = sizeof(*p->instructions) * (p_dry.length + 1); - p = kzalloc(sizeof(*p) + size, GFP_KERNEL); - if (!p) { - kfree(st); + p = kzalloc(struct_size(p, instructions, p_dry.length + 1), GFP_KERNEL); + if (!p) return -ENOMEM; - } - - ret = ida_alloc_range(&spi_engine->sync_ida, 0, U8_MAX, GFP_KERNEL); - if (ret < 0) { - kfree(p); - kfree(st); - return ret; - } - - st->sync_id = ret; spi_engine_compile_message(msg, false, p); - spi_engine_program_add_cmd(p, false, SPI_ENGINE_CMD_SYNC(st->sync_id)); + spi_engine_program_add_cmd(p, false, SPI_ENGINE_CMD_SYNC( + AXI_SPI_ENGINE_CUR_MSG_SYNC_ID)); - st->p = p; - st->cmd_buf = p->instructions; - st->cmd_length = p->length; - msg->state = st; + msg->opt_state = p; return 0; } -static int spi_engine_unprepare_message(struct spi_controller *host, - struct spi_message *msg) +static int spi_engine_unoptimize_message(struct spi_message *msg) { - struct spi_engine *spi_engine = spi_controller_get_devdata(host); - struct spi_engine_message_state *st = msg->state; - - ida_free(&spi_engine->sync_ida, st->sync_id); - kfree(st->p); - kfree(st); + kfree(msg->opt_state); return 0; } @@ -574,11 +534,18 @@ static int spi_engine_transfer_one_message(struct spi_controller *host, struct spi_message *msg) { struct spi_engine *spi_engine = spi_controller_get_devdata(host); - struct spi_engine_message_state *st = msg->state; + struct spi_engine_message_state *st = &spi_engine->msg_state; + struct spi_engine_program *p = msg->opt_state; unsigned int int_enable = 0; unsigned long flags; - mod_timer(&spi_engine->watchdog_timer, jiffies + msecs_to_jiffies(5000)); + /* reinitialize message state for this transfer */ + memset(st, 0, sizeof(*st)); + st->cmd_buf = p->instructions; + st->cmd_length = p->length; + msg->state = st; + + reinit_completion(&spi_engine->msg_complete); spin_lock_irqsave(&spi_engine->lock, flags); @@ -600,21 +567,16 @@ static int spi_engine_transfer_one_message(struct spi_controller *host, spi_engine->int_enable = int_enable; spin_unlock_irqrestore(&spi_engine->lock, flags); - return 0; -} + if (!wait_for_completion_timeout(&spi_engine->msg_complete, + msecs_to_jiffies(5000))) { + dev_err(&host->dev, + "Timeout occurred while waiting for transfer to complete. Hardware is probably broken.\n"); + msg->status = -ETIMEDOUT; + } -static void spi_engine_timeout(struct timer_list *timer) -{ - struct spi_engine *spi_engine = from_timer(spi_engine, timer, watchdog_timer); - struct spi_controller *host = spi_engine->controller; - - if (WARN_ON(!host->cur_msg)) - return; - - dev_err(&host->dev, - "Timeout occurred while waiting for transfer to complete. Hardware is probably broken.\n"); - host->cur_msg->status = -ETIMEDOUT; spi_finalize_current_message(host); + + return msg->status; } static void spi_engine_release_hw(void *p) @@ -645,9 +607,7 @@ static int spi_engine_probe(struct platform_device *pdev) spi_engine = spi_controller_get_devdata(host); spin_lock_init(&spi_engine->lock); - ida_init(&spi_engine->sync_ida); - timer_setup(&spi_engine->watchdog_timer, spi_engine_timeout, TIMER_IRQSAFE); - spi_engine->controller = host; + init_completion(&spi_engine->msg_complete); spi_engine->clk = devm_clk_get_enabled(&pdev->dev, "s_axi_aclk"); if (IS_ERR(spi_engine->clk)) @@ -661,12 +621,12 @@ static int spi_engine_probe(struct platform_device *pdev) if (IS_ERR(spi_engine->base)) return PTR_ERR(spi_engine->base); - version = readl(spi_engine->base + SPI_ENGINE_REG_VERSION); - if (SPI_ENGINE_VERSION_MAJOR(version) != 1) { + version = readl(spi_engine->base + ADI_AXI_REG_VERSION); + if (ADI_AXI_PCORE_VER_MAJOR(version) != 1) { dev_err(&pdev->dev, "Unsupported peripheral version %u.%u.%c\n", - SPI_ENGINE_VERSION_MAJOR(version), - SPI_ENGINE_VERSION_MINOR(version), - SPI_ENGINE_VERSION_PATCH(version)); + ADI_AXI_PCORE_VER_MAJOR(version), + ADI_AXI_PCORE_VER_MINOR(version), + ADI_AXI_PCORE_VER_PATCH(version)); return -ENODEV; } @@ -689,8 +649,8 @@ static int spi_engine_probe(struct platform_device *pdev) host->bits_per_word_mask = SPI_BPW_RANGE_MASK(1, 32); host->max_speed_hz = clk_get_rate(spi_engine->ref_clk) / 2; host->transfer_one_message = spi_engine_transfer_one_message; - host->prepare_message = spi_engine_prepare_message; - host->unprepare_message = spi_engine_unprepare_message; + host->optimize_message = spi_engine_optimize_message; + host->unoptimize_message = spi_engine_unoptimize_message; host->num_chipselect = 8; if (host->max_speed_hz == 0) diff --git a/drivers/spi/spi-bcm2835.c b/drivers/spi/spi-bcm2835.c index e709887eb2a9..e1b9b1235787 100644 --- a/drivers/spi/spi-bcm2835.c +++ b/drivers/spi/spi-bcm2835.c @@ -1117,19 +1117,6 @@ static int bcm2835_spi_prepare_message(struct spi_controller *ctlr, struct spi_device *spi = msg->spi; struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr); struct bcm2835_spidev *target = spi_get_ctldata(spi); - int ret; - - if (ctlr->can_dma) { - /* - * DMA transfers are limited to 16 bit (0 to 65535 bytes) by - * the SPI HW due to DLEN. Split up transfers (32-bit FIFO - * aligned) if the limit is exceeded. - */ - ret = spi_split_transfers_maxsize(ctlr, msg, 65532, - GFP_KERNEL | GFP_DMA); - if (ret) - return ret; - } /* * Set up clock polarity before spi_transfer_one_message() asserts @@ -1219,6 +1206,19 @@ static int bcm2835_spi_setup_dma(struct spi_controller *ctlr, return 0; } +static size_t bcm2835_spi_max_transfer_size(struct spi_device *spi) +{ + /* + * DMA transfers are limited to 16 bit (0 to 65535 bytes) by + * the SPI HW due to DLEN. Split up transfers (32-bit FIFO + * aligned) if the limit is exceeded. + */ + if (spi->controller->can_dma) + return 65532; + + return SIZE_MAX; +} + static int bcm2835_spi_setup(struct spi_device *spi) { struct spi_controller *ctlr = spi->controller; @@ -1348,6 +1348,7 @@ static int bcm2835_spi_probe(struct platform_device *pdev) ctlr->mode_bits = BCM2835_SPI_MODE_BITS; ctlr->bits_per_word_mask = SPI_BPW_MASK(8); ctlr->num_chipselect = 3; + ctlr->max_transfer_size = bcm2835_spi_max_transfer_size; ctlr->setup = bcm2835_spi_setup; ctlr->cleanup = bcm2835_spi_cleanup; ctlr->transfer_one = bcm2835_spi_transfer_one; diff --git a/drivers/spi/spi-bitbang.c b/drivers/spi/spi-bitbang.c index ecd44016c197..a0e2204fc039 100644 --- a/drivers/spi/spi-bitbang.c +++ b/drivers/spi/spi-bitbang.c @@ -187,7 +187,7 @@ int spi_bitbang_setup(struct spi_device *spi) bool initial_setup = false; int retval; - bitbang = spi_master_get_devdata(spi->master); + bitbang = spi_controller_get_devdata(spi->controller); if (!cs) { cs = kzalloc(sizeof(*cs), GFP_KERNEL); @@ -236,7 +236,7 @@ static int spi_bitbang_bufs(struct spi_device *spi, struct spi_transfer *t) unsigned nsecs = cs->nsecs; struct spi_bitbang *bitbang; - bitbang = spi_master_get_devdata(spi->master); + bitbang = spi_controller_get_devdata(spi->controller); if (bitbang->set_line_direction) { int err; @@ -268,11 +268,11 @@ static int spi_bitbang_bufs(struct spi_device *spi, struct spi_transfer *t) * transfer-at-a-time ones to leverage dma or fifo hardware. */ -static int spi_bitbang_prepare_hardware(struct spi_master *spi) +static int spi_bitbang_prepare_hardware(struct spi_controller *spi) { struct spi_bitbang *bitbang; - bitbang = spi_master_get_devdata(spi); + bitbang = spi_controller_get_devdata(spi); mutex_lock(&bitbang->lock); bitbang->busy = 1; @@ -281,11 +281,11 @@ static int spi_bitbang_prepare_hardware(struct spi_master *spi) return 0; } -static int spi_bitbang_transfer_one(struct spi_master *master, +static int spi_bitbang_transfer_one(struct spi_controller *ctlr, struct spi_device *spi, struct spi_transfer *transfer) { - struct spi_bitbang *bitbang = spi_master_get_devdata(master); + struct spi_bitbang *bitbang = spi_controller_get_devdata(ctlr); int status = 0; if (bitbang->setup_transfer) { @@ -303,16 +303,16 @@ static int spi_bitbang_transfer_one(struct spi_master *master, status = -EREMOTEIO; out: - spi_finalize_current_transfer(master); + spi_finalize_current_transfer(ctlr); return status; } -static int spi_bitbang_unprepare_hardware(struct spi_master *spi) +static int spi_bitbang_unprepare_hardware(struct spi_controller *spi) { struct spi_bitbang *bitbang; - bitbang = spi_master_get_devdata(spi); + bitbang = spi_controller_get_devdata(spi); mutex_lock(&bitbang->lock); bitbang->busy = 0; @@ -323,7 +323,7 @@ static int spi_bitbang_unprepare_hardware(struct spi_master *spi) static void spi_bitbang_set_cs(struct spi_device *spi, bool enable) { - struct spi_bitbang *bitbang = spi_master_get_devdata(spi->master); + struct spi_bitbang *bitbang = spi_controller_get_devdata(spi->controller); /* SPI core provides CS high / low, but bitbang driver * expects CS active @@ -341,10 +341,10 @@ static void spi_bitbang_set_cs(struct spi_device *spi, bool enable) int spi_bitbang_init(struct spi_bitbang *bitbang) { - struct spi_master *master = bitbang->master; + struct spi_controller *ctlr = bitbang->ctlr; bool custom_cs; - if (!master) + if (!ctlr) return -EINVAL; /* * We only need the chipselect callback if we are actually using it. @@ -352,39 +352,39 @@ int spi_bitbang_init(struct spi_bitbang *bitbang) * SPI_CONTROLLER_GPIO_SS flag is set, we always need to call the * driver-specific chipselect routine. */ - custom_cs = (!master->use_gpio_descriptors || - (master->flags & SPI_CONTROLLER_GPIO_SS)); + custom_cs = (!ctlr->use_gpio_descriptors || + (ctlr->flags & SPI_CONTROLLER_GPIO_SS)); if (custom_cs && !bitbang->chipselect) return -EINVAL; mutex_init(&bitbang->lock); - if (!master->mode_bits) - master->mode_bits = SPI_CPOL | SPI_CPHA | bitbang->flags; + if (!ctlr->mode_bits) + ctlr->mode_bits = SPI_CPOL | SPI_CPHA | bitbang->flags; - if (master->transfer || master->transfer_one_message) + if (ctlr->transfer || ctlr->transfer_one_message) return -EINVAL; - master->prepare_transfer_hardware = spi_bitbang_prepare_hardware; - master->unprepare_transfer_hardware = spi_bitbang_unprepare_hardware; - master->transfer_one = spi_bitbang_transfer_one; + ctlr->prepare_transfer_hardware = spi_bitbang_prepare_hardware; + ctlr->unprepare_transfer_hardware = spi_bitbang_unprepare_hardware; + ctlr->transfer_one = spi_bitbang_transfer_one; /* * When using GPIO descriptors, the ->set_cs() callback doesn't even * get called unless SPI_CONTROLLER_GPIO_SS is set. */ if (custom_cs) - master->set_cs = spi_bitbang_set_cs; + ctlr->set_cs = spi_bitbang_set_cs; if (!bitbang->txrx_bufs) { bitbang->use_dma = 0; bitbang->txrx_bufs = spi_bitbang_bufs; - if (!master->setup) { + if (!ctlr->setup) { if (!bitbang->setup_transfer) bitbang->setup_transfer = spi_bitbang_setup_transfer; - master->setup = spi_bitbang_setup; - master->cleanup = spi_bitbang_cleanup; + ctlr->setup = spi_bitbang_setup; + ctlr->cleanup = spi_bitbang_cleanup; } } @@ -411,18 +411,18 @@ EXPORT_SYMBOL_GPL(spi_bitbang_init); * master methods. Those methods are the defaults if the bitbang->txrx_bufs * routine isn't initialized. * - * This routine registers the spi_master, which will process requests in a + * This routine registers the spi_controller, which will process requests in a * dedicated task, keeping IRQs unblocked most of the time. To stop * processing those requests, call spi_bitbang_stop(). * - * On success, this routine will take a reference to master. The caller is - * responsible for calling spi_bitbang_stop() to decrement the reference and - * spi_master_put() as counterpart of spi_alloc_master() to prevent a memory + * On success, this routine will take a reference to the controller. The caller + * is responsible for calling spi_bitbang_stop() to decrement the reference and + * spi_controller_put() as counterpart of spi_alloc_master() to prevent a memory * leak. */ int spi_bitbang_start(struct spi_bitbang *bitbang) { - struct spi_master *master = bitbang->master; + struct spi_controller *ctlr = bitbang->ctlr; int ret; ret = spi_bitbang_init(bitbang); @@ -432,9 +432,9 @@ int spi_bitbang_start(struct spi_bitbang *bitbang) /* driver may get busy before register() returns, especially * if someone registered boardinfo for devices */ - ret = spi_register_master(spi_master_get(master)); + ret = spi_register_controller(spi_controller_get(ctlr)); if (ret) - spi_master_put(master); + spi_controller_put(ctlr); return ret; } @@ -445,7 +445,7 @@ EXPORT_SYMBOL_GPL(spi_bitbang_start); */ void spi_bitbang_stop(struct spi_bitbang *bitbang) { - spi_unregister_master(bitbang->master); + spi_unregister_controller(bitbang->ctlr); } EXPORT_SYMBOL_GPL(spi_bitbang_stop); diff --git a/drivers/spi/spi-butterfly.c b/drivers/spi/spi-butterfly.c index 289b4454242a..1d267e6c22a4 100644 --- a/drivers/spi/spi-butterfly.c +++ b/drivers/spi/spi-butterfly.c @@ -205,7 +205,7 @@ static void butterfly_attach(struct parport *p) host->bus_num = 42; host->num_chipselect = 2; - pp->bitbang.master = host; + pp->bitbang.ctlr = host; pp->bitbang.chipselect = butterfly_chipselect; pp->bitbang.txrx_word[SPI_MODE_0] = butterfly_txrx_word_mode0; @@ -263,7 +263,7 @@ static void butterfly_attach(struct parport *p) pp->info[0].platform_data = &flash; pp->info[0].chip_select = 1; pp->info[0].controller_data = pp; - pp->dataflash = spi_new_device(pp->bitbang.master, &pp->info[0]); + pp->dataflash = spi_new_device(pp->bitbang.ctlr, &pp->info[0]); if (pp->dataflash) pr_debug("%s: dataflash at %s\n", p->name, dev_name(&pp->dataflash->dev)); @@ -308,7 +308,7 @@ static void butterfly_detach(struct parport *p) parport_release(pp->pd); parport_unregister_device(pp->pd); - spi_controller_put(pp->bitbang.master); + spi_controller_put(pp->bitbang.ctlr); } static struct parport_driver butterfly_driver = { diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c index 1a8d03958dff..350b3dab3a05 100644 --- a/drivers/spi/spi-cadence-quadspi.c +++ b/drivers/spi/spi-cadence-quadspi.c @@ -31,7 +31,9 @@ #include #define CQSPI_NAME "cadence-qspi" -#define CQSPI_MAX_CHIPSELECT 16 +#define CQSPI_MAX_CHIPSELECT 4 + +static_assert(CQSPI_MAX_CHIPSELECT <= SPI_CS_CNT_MAX); /* Quirks */ #define CQSPI_NEEDS_WR_DELAY BIT(0) @@ -1410,7 +1412,7 @@ static int cqspi_mem_process(struct spi_mem *mem, const struct spi_mem_op *op) static int cqspi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op) { int ret; - struct cqspi_st *cqspi = spi_master_get_devdata(mem->spi->master); + struct cqspi_st *cqspi = spi_controller_get_devdata(mem->spi->controller); struct device *dev = &cqspi->pdev->dev; ret = pm_runtime_resume_and_get(dev); @@ -1619,6 +1621,7 @@ static const struct spi_controller_mem_caps cqspi_mem_caps = { static int cqspi_setup_flash(struct cqspi_st *cqspi) { + unsigned int max_cs = cqspi->num_chipselect - 1; struct platform_device *pdev = cqspi->pdev; struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; @@ -1635,10 +1638,12 @@ static int cqspi_setup_flash(struct cqspi_st *cqspi) return ret; } - if (cs >= CQSPI_MAX_CHIPSELECT) { + if (cs >= cqspi->num_chipselect) { dev_err(dev, "Chip select %d out of range.\n", cs); of_node_put(np); return -EINVAL; + } else if (cs < max_cs) { + max_cs = cs; } f_pdata = &cqspi->f_pdata[cs]; @@ -1652,6 +1657,7 @@ static int cqspi_setup_flash(struct cqspi_st *cqspi) } } + cqspi->num_chipselect = max_cs + 1; return 0; } @@ -1712,10 +1718,9 @@ static int cqspi_probe(struct platform_device *pdev) int irq; host = devm_spi_alloc_host(&pdev->dev, sizeof(*cqspi)); - if (!host) { - dev_err(&pdev->dev, "devm_spi_alloc_host failed\n"); + if (!host) return -ENOMEM; - } + host->mode_bits = SPI_RX_QUAD | SPI_RX_DUAL; host->mem_ops = &cqspi_mem_ops; host->mem_caps = &cqspi_mem_caps; @@ -1863,14 +1868,14 @@ static int cqspi_probe(struct platform_device *pdev) cqspi->current_cs = -1; cqspi->sclk = 0; - host->num_chipselect = cqspi->num_chipselect; - ret = cqspi_setup_flash(cqspi); if (ret) { dev_err(dev, "failed to setup flash parameters %d\n", ret); goto probe_setup_failed; } + host->num_chipselect = cqspi->num_chipselect; + if (cqspi->use_direct_mode) { ret = cqspi_request_mmap_dma(cqspi); if (ret == -EPROBE_DEFER) diff --git a/drivers/spi/spi-cavium.c b/drivers/spi/spi-cavium.c index dfe224defd6e..26b8cd1c76e1 100644 --- a/drivers/spi/spi-cavium.c +++ b/drivers/spi/spi-cavium.c @@ -124,10 +124,10 @@ static int octeon_spi_do_transfer(struct octeon_spi *p, return xfer->len; } -int octeon_spi_transfer_one_message(struct spi_master *master, +int octeon_spi_transfer_one_message(struct spi_controller *ctlr, struct spi_message *msg) { - struct octeon_spi *p = spi_master_get_devdata(master); + struct octeon_spi *p = spi_controller_get_devdata(ctlr); unsigned int total_len = 0; int status = 0; struct spi_transfer *xfer; @@ -145,6 +145,6 @@ int octeon_spi_transfer_one_message(struct spi_master *master, err: msg->status = status; msg->actual_length = total_len; - spi_finalize_current_message(master); + spi_finalize_current_message(ctlr); return status; } diff --git a/drivers/spi/spi-cavium.h b/drivers/spi/spi-cavium.h index 1f3ac463a20b..af53a0c31476 100644 --- a/drivers/spi/spi-cavium.h +++ b/drivers/spi/spi-cavium.h @@ -28,7 +28,7 @@ struct octeon_spi { #define OCTEON_SPI_TX(x) (x->regs.tx) #define OCTEON_SPI_DAT0(x) (x->regs.data) -int octeon_spi_transfer_one_message(struct spi_master *master, +int octeon_spi_transfer_one_message(struct spi_controller *ctlr, struct spi_message *msg); /* MPI register descriptions */ diff --git a/drivers/spi/spi-cs42l43.c b/drivers/spi/spi-cs42l43.c index adf19e8c4c8a..27c995b657f2 100644 --- a/drivers/spi/spi-cs42l43.c +++ b/drivers/spi/spi-cs42l43.c @@ -201,6 +201,11 @@ static size_t cs42l43_spi_max_length(struct spi_device *spi) return CS42L43_SPI_MAX_LENGTH; } +static void cs42l43_release_of_node(void *data) +{ + fwnode_handle_put(data); +} + static int cs42l43_spi_probe(struct platform_device *pdev) { struct cs42l43 *cs42l43 = dev_get_drvdata(pdev->dev.parent); @@ -227,12 +232,6 @@ static int cs42l43_spi_probe(struct platform_device *pdev) priv->ctlr->transfer_one = cs42l43_transfer_one; priv->ctlr->set_cs = cs42l43_set_cs; priv->ctlr->max_transfer_size = cs42l43_spi_max_length; - - if (is_of_node(fwnode)) - fwnode = fwnode_get_named_child_node(fwnode, "spi"); - - device_set_node(&priv->ctlr->dev, fwnode); - priv->ctlr->mode_bits = SPI_3WIRE | SPI_MODE_X_MASK; priv->ctlr->flags = SPI_CONTROLLER_HALF_DUPLEX; priv->ctlr->bits_per_word_mask = SPI_BPW_MASK(8) | SPI_BPW_MASK(16) | @@ -256,6 +255,17 @@ static int cs42l43_spi_probe(struct platform_device *pdev) regmap_write(priv->regmap, CS42L43_SPI_CONFIG3, 0); regmap_write(priv->regmap, CS42L43_SPI_CONFIG4, CS42L43_SPI_STALL_ENA_MASK); + if (is_of_node(fwnode)) { + fwnode = fwnode_get_named_child_node(fwnode, "spi"); + ret = devm_add_action(priv->dev, cs42l43_release_of_node, fwnode); + if (ret) { + fwnode_handle_put(fwnode); + return ret; + } + } + + device_set_node(&priv->ctlr->dev, fwnode); + ret = devm_spi_register_controller(priv->dev, priv->ctlr); if (ret) { dev_err(priv->dev, "Failed to register SPI controller: %d\n", ret); diff --git a/drivers/spi/spi-davinci.c b/drivers/spi/spi-davinci.c index 5688be245c68..be3998104bfb 100644 --- a/drivers/spi/spi-davinci.c +++ b/drivers/spi/spi-davinci.c @@ -459,7 +459,7 @@ static bool davinci_spi_can_dma(struct spi_controller *host, static int davinci_spi_check_error(struct davinci_spi *dspi, int int_status) { - struct device *sdev = dspi->bitbang.master->dev.parent; + struct device *sdev = dspi->bitbang.ctlr->dev.parent; if (int_status & SPIFLG_TIMEOUT_MASK) { dev_err(sdev, "SPI Time-out Error\n"); @@ -742,7 +742,7 @@ static irqreturn_t davinci_spi_irq(s32 irq, void *data) static int davinci_spi_request_dma(struct davinci_spi *dspi) { - struct device *sdev = dspi->bitbang.master->dev.parent; + struct device *sdev = dspi->bitbang.ctlr->dev.parent; dspi->dma_rx = dma_request_chan(sdev, "rx"); if (IS_ERR(dspi->dma_rx)) @@ -913,7 +913,7 @@ static int davinci_spi_probe(struct platform_device *pdev) if (ret) goto free_host; - dspi->bitbang.master = host; + dspi->bitbang.ctlr = host; dspi->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(dspi->clk)) { diff --git a/drivers/spi/spi-dw-dma.c b/drivers/spi/spi-dw-dma.c index 0ecbb6c36e23..f4c209e5f52b 100644 --- a/drivers/spi/spi-dw-dma.c +++ b/drivers/spi/spi-dw-dma.c @@ -577,7 +577,7 @@ static int dw_spi_dma_transfer_one(struct dw_spi *dws, sg_init_table(&tx_tmp, 1); sg_init_table(&rx_tmp, 1); - for (base = 0, len = 0; base < xfer->len; base += len) { + for (base = 0; base < xfer->len; base += len) { /* Fetch next Tx DMA data chunk */ if (!tx_len) { tx_sg = !tx_sg ? &xfer->tx_sg.sgl[0] : sg_next(tx_sg); diff --git a/drivers/spi/spi-fsl-dspi.c b/drivers/spi/spi-fsl-dspi.c index c9eae046f66c..38defdcf9370 100644 --- a/drivers/spi/spi-fsl-dspi.c +++ b/drivers/spi/spi-fsl-dspi.c @@ -502,15 +502,12 @@ static int dspi_request_dma(struct fsl_dspi *dspi, phys_addr_t phy_addr) return -ENOMEM; dma->chan_rx = dma_request_chan(dev, "rx"); - if (IS_ERR(dma->chan_rx)) { - return dev_err_probe(dev, PTR_ERR(dma->chan_rx), - "rx dma channel not available\n"); - } + if (IS_ERR(dma->chan_rx)) + return dev_err_probe(dev, PTR_ERR(dma->chan_rx), "rx dma channel not available\n"); dma->chan_tx = dma_request_chan(dev, "tx"); if (IS_ERR(dma->chan_tx)) { - ret = PTR_ERR(dma->chan_tx); - dev_err_probe(dev, ret, "tx dma channel not available\n"); + ret = dev_err_probe(dev, PTR_ERR(dma->chan_tx), "tx dma channel not available\n"); goto err_tx_channel; } @@ -541,16 +538,14 @@ static int dspi_request_dma(struct fsl_dspi *dspi, phys_addr_t phy_addr) cfg.direction = DMA_DEV_TO_MEM; ret = dmaengine_slave_config(dma->chan_rx, &cfg); if (ret) { - dev_err(dev, "can't configure rx dma channel\n"); - ret = -EINVAL; + dev_err_probe(dev, ret, "can't configure rx dma channel\n"); goto err_slave_config; } cfg.direction = DMA_MEM_TO_DEV; ret = dmaengine_slave_config(dma->chan_tx, &cfg); if (ret) { - dev_err(dev, "can't configure tx dma channel\n"); - ret = -EINVAL; + dev_err_probe(dev, ret, "can't configure tx dma channel\n"); goto err_slave_config; } diff --git a/drivers/spi/spi-fsl-lib.c b/drivers/spi/spi-fsl-lib.c index 885757c29fbb..4fc2c56555b5 100644 --- a/drivers/spi/spi-fsl-lib.c +++ b/drivers/spi/spi-fsl-lib.c @@ -82,18 +82,18 @@ void mpc8xxx_spi_probe(struct device *dev, struct resource *mem, unsigned int irq) { struct fsl_spi_platform_data *pdata = dev_get_platdata(dev); - struct spi_master *master; + struct spi_controller *ctlr; struct mpc8xxx_spi *mpc8xxx_spi; - master = dev_get_drvdata(dev); + ctlr = dev_get_drvdata(dev); /* the spi->mode bits understood by this driver: */ - master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH + ctlr->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH | SPI_LSB_FIRST | SPI_LOOP; - master->dev.of_node = dev->of_node; + ctlr->dev.of_node = dev->of_node; - mpc8xxx_spi = spi_master_get_devdata(master); + mpc8xxx_spi = spi_controller_get_devdata(ctlr); mpc8xxx_spi->dev = dev; mpc8xxx_spi->get_rx = mpc8xxx_spi_rx_buf_u8; mpc8xxx_spi->get_tx = mpc8xxx_spi_tx_buf_u8; @@ -104,8 +104,8 @@ void mpc8xxx_spi_probe(struct device *dev, struct resource *mem, mpc8xxx_spi->rx_shift = 0; mpc8xxx_spi->tx_shift = 0; - master->bus_num = pdata->bus_num; - master->num_chipselect = pdata->max_chipselect; + ctlr->bus_num = pdata->bus_num; + ctlr->num_chipselect = pdata->max_chipselect; init_completion(&mpc8xxx_spi->done); } diff --git a/drivers/spi/spi-geni-qcom.c b/drivers/spi/spi-geni-qcom.c index 15f84e68d4d2..37ef8c40b276 100644 --- a/drivers/spi/spi-geni-qcom.c +++ b/drivers/spi/spi-geni-qcom.c @@ -647,7 +647,7 @@ static void spi_geni_release_dma_chan(struct spi_geni_master *mas) static int spi_geni_init(struct spi_geni_master *mas) { - struct spi_master *spi = dev_get_drvdata(mas->dev); + struct spi_controller *spi = dev_get_drvdata(mas->dev); struct geni_se *se = &mas->se; unsigned int proto, major, minor, ver; u32 spi_tx_cfg, fifo_disable; diff --git a/drivers/spi/spi-gpio.c b/drivers/spi/spi-gpio.c index d8db4564b406..909cce109bba 100644 --- a/drivers/spi/spi-gpio.c +++ b/drivers/spi/spi-gpio.c @@ -427,7 +427,7 @@ static int spi_gpio_probe(struct platform_device *pdev) host->cleanup = spi_gpio_cleanup; bb = &spi_gpio->bitbang; - bb->master = host; + bb->ctlr = host; /* * There is some additional business, apart from driving the CS GPIO * line, that we need to do on selection. This makes the local diff --git a/drivers/spi/spi-intel.c b/drivers/spi/spi-intel.c index 3654ae35d2db..3e5dcf2b3c8a 100644 --- a/drivers/spi/spi-intel.c +++ b/drivers/spi/spi-intel.c @@ -1254,6 +1254,13 @@ static void intel_spi_fill_partition(struct intel_spi *ispi, if (end > part->size) part->size = end; } + + /* + * Regions can refer to the second chip too so in this case we + * just make the BIOS partition to occupy the whole chip. + */ + if (ispi->chip0_size && part->size > ispi->chip0_size) + part->size = MTDPART_SIZ_FULL; } static int intel_spi_read_desc(struct intel_spi *ispi) @@ -1346,9 +1353,14 @@ static int intel_spi_read_desc(struct intel_spi *ispi) static int intel_spi_populate_chip(struct intel_spi *ispi) { struct flash_platform_data *pdata; + struct mtd_partition *parts; struct spi_board_info chip; int ret; + ret = intel_spi_read_desc(ispi); + if (ret) + return ret; + pdata = devm_kzalloc(ispi->dev, sizeof(*pdata), GFP_KERNEL); if (!pdata) return -ENOMEM; @@ -1368,15 +1380,27 @@ static int intel_spi_populate_chip(struct intel_spi *ispi) if (!spi_new_device(ispi->host, &chip)) return -ENODEV; - ret = intel_spi_read_desc(ispi); - if (ret) - return ret; - /* Add the second chip if present */ if (ispi->host->num_chipselect < 2) return 0; - chip.platform_data = NULL; + pdata = devm_kzalloc(ispi->dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) + return -ENOMEM; + + pdata->name = devm_kasprintf(ispi->dev, GFP_KERNEL, "%s-chip1", + dev_name(ispi->dev)); + pdata->nr_parts = 1; + parts = devm_kcalloc(ispi->dev, pdata->nr_parts, sizeof(*parts), + GFP_KERNEL); + if (!parts) + return -ENOMEM; + + parts[0].size = MTDPART_SIZ_FULL; + parts[0].name = "BIOS1"; + pdata->parts = parts; + + chip.platform_data = pdata; chip.chip_select = 1; if (!spi_new_device(ispi->host, &chip)) diff --git a/drivers/spi/spi-lm70llp.c b/drivers/spi/spi-lm70llp.c index e6a65b5c8c31..f982bdebd028 100644 --- a/drivers/spi/spi-lm70llp.c +++ b/drivers/spi/spi-lm70llp.c @@ -212,7 +212,7 @@ static void spi_lm70llp_attach(struct parport *p) /* * SPI and bitbang hookup. */ - pp->bitbang.master = host; + pp->bitbang.ctlr = host; pp->bitbang.chipselect = lm70_chipselect; pp->bitbang.txrx_word[SPI_MODE_0] = lm70_txrx; pp->bitbang.flags = SPI_3WIRE; @@ -264,7 +264,7 @@ static void spi_lm70llp_attach(struct parport *p) * the board info's (void *)controller_data. */ pp->info.controller_data = pp; - pp->spidev_lm70 = spi_new_device(pp->bitbang.master, &pp->info); + pp->spidev_lm70 = spi_new_device(pp->bitbang.ctlr, &pp->info); if (pp->spidev_lm70) dev_dbg(&pp->spidev_lm70->dev, "spidev_lm70 at %s\n", dev_name(&pp->spidev_lm70->dev)); @@ -309,7 +309,7 @@ static void spi_lm70llp_detach(struct parport *p) parport_release(pp->pd); parport_unregister_device(pp->pd); - spi_controller_put(pp->bitbang.master); + spi_controller_put(pp->bitbang.ctlr); lm70llp = NULL; } diff --git a/drivers/spi/spi-loopback-test.c b/drivers/spi/spi-loopback-test.c index bbf2015d8e5c..fee8893d2751 100644 --- a/drivers/spi/spi-loopback-test.c +++ b/drivers/spi/spi-loopback-test.c @@ -1031,8 +1031,8 @@ int spi_test_run_test(struct spi_device *spi, const struct spi_test *test, #define FOR_EACH_ALIGNMENT(var) \ for (var = 0; \ var < (test->iterate_##var ? \ - (spi->master->dma_alignment ? \ - spi->master->dma_alignment : \ + (spi->controller->dma_alignment ? \ + spi->controller->dma_alignment : \ test->iterate_##var) : \ 1); \ var++) diff --git a/drivers/spi/spi-mem.c b/drivers/spi/spi-mem.c index 2dc8ceb85374..c9d6d42a88f5 100644 --- a/drivers/spi/spi-mem.c +++ b/drivers/spi/spi-mem.c @@ -297,6 +297,49 @@ static void spi_mem_access_end(struct spi_mem *mem) pm_runtime_put(ctlr->dev.parent); } +static void spi_mem_add_op_stats(struct spi_statistics __percpu *pcpu_stats, + const struct spi_mem_op *op, int exec_op_ret) +{ + struct spi_statistics *stats; + u64 len, l2len; + + get_cpu(); + stats = this_cpu_ptr(pcpu_stats); + u64_stats_update_begin(&stats->syncp); + + /* + * We do not have the concept of messages or transfers. Let's consider + * that one operation is equivalent to one message and one transfer. + */ + u64_stats_inc(&stats->messages); + u64_stats_inc(&stats->transfers); + + /* Use the sum of all lengths as bytes count and histogram value. */ + len = op->cmd.nbytes + op->addr.nbytes; + len += op->dummy.nbytes + op->data.nbytes; + u64_stats_add(&stats->bytes, len); + l2len = min(fls(len), SPI_STATISTICS_HISTO_SIZE) - 1; + u64_stats_inc(&stats->transfer_bytes_histo[l2len]); + + /* Only account for data bytes as transferred bytes. */ + if (op->data.nbytes && op->data.dir == SPI_MEM_DATA_OUT) + u64_stats_add(&stats->bytes_tx, op->data.nbytes); + if (op->data.nbytes && op->data.dir == SPI_MEM_DATA_IN) + u64_stats_add(&stats->bytes_rx, op->data.nbytes); + + /* + * A timeout is not an error, following the same behavior as + * spi_transfer_one_message(). + */ + if (exec_op_ret == -ETIMEDOUT) + u64_stats_inc(&stats->timedout); + else if (exec_op_ret) + u64_stats_inc(&stats->errors); + + u64_stats_update_end(&stats->syncp); + put_cpu(); +} + /** * spi_mem_exec_op() - Execute a memory operation * @mem: the SPI memory @@ -339,8 +382,12 @@ int spi_mem_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) * read path) and expect the core to use the regular SPI * interface in other cases. */ - if (!ret || ret != -ENOTSUPP || ret != -EOPNOTSUPP) + if (!ret || ret != -ENOTSUPP || ret != -EOPNOTSUPP) { + spi_mem_add_op_stats(ctlr->pcpu_statistics, op, ret); + spi_mem_add_op_stats(mem->spi->pcpu_statistics, op, ret); + return ret; + } } tmpbufsize = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes; diff --git a/drivers/spi/spi-mt65xx.c b/drivers/spi/spi-mt65xx.c index 8d5d170d49cc..8d4633b353ee 100644 --- a/drivers/spi/spi-mt65xx.c +++ b/drivers/spi/spi-mt65xx.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -1316,6 +1317,8 @@ static int mtk_spi_suspend(struct device *dev) clk_disable_unprepare(mdata->spi_hclk); } + pinctrl_pm_select_sleep_state(dev); + return 0; } @@ -1325,6 +1328,8 @@ static int mtk_spi_resume(struct device *dev) struct spi_controller *host = dev_get_drvdata(dev); struct mtk_spi *mdata = spi_controller_get_devdata(host); + pinctrl_pm_select_default_state(dev); + if (!pm_runtime_suspended(dev)) { ret = clk_prepare_enable(mdata->spi_clk); if (ret < 0) { diff --git a/drivers/spi/spi-nxp-fspi.c b/drivers/spi/spi-nxp-fspi.c index e13f678f2395..88397f712a3b 100644 --- a/drivers/spi/spi-nxp-fspi.c +++ b/drivers/spi/spi-nxp-fspi.c @@ -591,7 +591,7 @@ static void nxp_fspi_prepare_lut(struct nxp_fspi *f, for (i = 0; i < ARRAY_SIZE(lutval); i++) fspi_writel(f, lutval[i], base + FSPI_LUT_REG(i)); - dev_dbg(f->dev, "CMD[%x] lutval[0:%x \t 1:%x \t 2:%x \t 3:%x], size: 0x%08x\n", + dev_dbg(f->dev, "CMD[%02x] lutval[0:%08x 1:%08x 2:%08x 3:%08x], size: 0x%08x\n", op->cmd.opcode, lutval[0], lutval[1], lutval[2], lutval[3], op->data.nbytes); /* lock LUT */ diff --git a/drivers/spi/spi-oc-tiny.c b/drivers/spi/spi-oc-tiny.c index cf7c111088a6..6ea38f5e7d64 100644 --- a/drivers/spi/spi-oc-tiny.c +++ b/drivers/spi/spi-oc-tiny.c @@ -194,7 +194,7 @@ static int tiny_spi_of_probe(struct platform_device *pdev) if (!np) return 0; - hw->bitbang.master->dev.of_node = pdev->dev.of_node; + hw->bitbang.ctlr->dev.of_node = pdev->dev.of_node; if (!of_property_read_u32(np, "clock-frequency", &val)) hw->freq = val; if (!of_property_read_u32(np, "baud-width", &val)) @@ -229,7 +229,7 @@ static int tiny_spi_probe(struct platform_device *pdev) platform_set_drvdata(pdev, hw); /* setup the state for the bitbang driver */ - hw->bitbang.master = host; + hw->bitbang.ctlr = host; hw->bitbang.setup_transfer = tiny_spi_setup_transfer; hw->bitbang.txrx_bufs = tiny_spi_txrx_bufs; @@ -274,7 +274,7 @@ exit: static void tiny_spi_remove(struct platform_device *pdev) { struct tiny_spi *hw = platform_get_drvdata(pdev); - struct spi_controller *host = hw->bitbang.master; + struct spi_controller *host = hw->bitbang.ctlr; spi_bitbang_stop(&hw->bitbang); spi_controller_put(host); diff --git a/drivers/spi/spi-omap-uwire.c b/drivers/spi/spi-omap-uwire.c index 2dd1c1bcf4bf..210a98d903fa 100644 --- a/drivers/spi/spi-omap-uwire.c +++ b/drivers/spi/spi-omap-uwire.c @@ -448,7 +448,7 @@ static void uwire_off(struct uwire_spi *uwire) { uwire_write_reg(UWIRE_SR3, 0); clk_disable_unprepare(uwire->ck); - spi_controller_put(uwire->bitbang.master); + spi_controller_put(uwire->bitbang.ctlr); } static int uwire_probe(struct platform_device *pdev) @@ -493,7 +493,7 @@ static int uwire_probe(struct platform_device *pdev) host->setup = uwire_setup; host->cleanup = uwire_cleanup; - uwire->bitbang.master = host; + uwire->bitbang.ctlr = host; uwire->bitbang.chipselect = uwire_chipselect; uwire->bitbang.setup_transfer = uwire_setup_transfer; uwire->bitbang.txrx_bufs = uwire_txrx; diff --git a/drivers/spi/spi-pci1xxxx.c b/drivers/spi/spi-pci1xxxx.c index 5b2d3e4e21b7..969965d7bc98 100644 --- a/drivers/spi/spi-pci1xxxx.c +++ b/drivers/spi/spi-pci1xxxx.c @@ -5,8 +5,15 @@ // Kumaravel Thiagarajan +#include +#include +#include +#include #include +#include +#include #include +#include #include #include @@ -32,8 +39,68 @@ #define SPI_MST_CTL_MODE_SEL (BIT(2)) #define SPI_MST_CTL_GO (BIT(0)) +#define SPI_PERI_ADDR_BASE (0x160000) +#define SPI_SYSTEM_ADDR_BASE (0x2000) #define SPI_MST1_ADDR_BASE (0x800) +#define DEV_REV_REG (SPI_SYSTEM_ADDR_BASE + 0x00) +#define SPI_SYSLOCK_REG (SPI_SYSTEM_ADDR_BASE + 0xA0) +#define SPI_CONFIG_PERI_ENABLE_REG (SPI_SYSTEM_ADDR_BASE + 0x108) + +#define SPI_PERI_ENBLE_PF_MASK (GENMASK(17, 16)) +#define DEV_REV_MASK (GENMASK(7, 0)) + +#define SPI_SYSLOCK BIT(4) +#define SPI0 (0) +#define SPI1 (1) + +/* DMA Related Registers */ +#define SPI_DMA_ADDR_BASE (0x1000) +#define SPI_DMA_GLOBAL_WR_ENGINE_EN (SPI_DMA_ADDR_BASE + 0x0C) +#define SPI_DMA_WR_DOORBELL_REG (SPI_DMA_ADDR_BASE + 0x10) +#define SPI_DMA_GLOBAL_RD_ENGINE_EN (SPI_DMA_ADDR_BASE + 0x2C) +#define SPI_DMA_RD_DOORBELL_REG (SPI_DMA_ADDR_BASE + 0x30) +#define SPI_DMA_INTR_WR_STS (SPI_DMA_ADDR_BASE + 0x4C) +#define SPI_DMA_WR_INT_MASK (SPI_DMA_ADDR_BASE + 0x54) +#define SPI_DMA_INTR_WR_CLR (SPI_DMA_ADDR_BASE + 0x58) +#define SPI_DMA_ERR_WR_STS (SPI_DMA_ADDR_BASE + 0x5C) +#define SPI_DMA_INTR_IMWR_WDONE_LOW (SPI_DMA_ADDR_BASE + 0x60) +#define SPI_DMA_INTR_IMWR_WDONE_HIGH (SPI_DMA_ADDR_BASE + 0x64) +#define SPI_DMA_INTR_IMWR_WABORT_LOW (SPI_DMA_ADDR_BASE + 0x68) +#define SPI_DMA_INTR_IMWR_WABORT_HIGH (SPI_DMA_ADDR_BASE + 0x6C) +#define SPI_DMA_INTR_WR_IMWR_DATA (SPI_DMA_ADDR_BASE + 0x70) +#define SPI_DMA_INTR_RD_STS (SPI_DMA_ADDR_BASE + 0xA0) +#define SPI_DMA_RD_INT_MASK (SPI_DMA_ADDR_BASE + 0xA8) +#define SPI_DMA_INTR_RD_CLR (SPI_DMA_ADDR_BASE + 0xAC) +#define SPI_DMA_ERR_RD_STS (SPI_DMA_ADDR_BASE + 0xB8) +#define SPI_DMA_INTR_IMWR_RDONE_LOW (SPI_DMA_ADDR_BASE + 0xCC) +#define SPI_DMA_INTR_IMWR_RDONE_HIGH (SPI_DMA_ADDR_BASE + 0xD0) +#define SPI_DMA_INTR_IMWR_RABORT_LOW (SPI_DMA_ADDR_BASE + 0xD4) +#define SPI_DMA_INTR_IMWR_RABORT_HIGH (SPI_DMA_ADDR_BASE + 0xD8) +#define SPI_DMA_INTR_RD_IMWR_DATA (SPI_DMA_ADDR_BASE + 0xDC) + +#define SPI_DMA_CH0_WR_BASE (SPI_DMA_ADDR_BASE + 0x200) +#define SPI_DMA_CH0_RD_BASE (SPI_DMA_ADDR_BASE + 0x300) +#define SPI_DMA_CH1_WR_BASE (SPI_DMA_ADDR_BASE + 0x400) +#define SPI_DMA_CH1_RD_BASE (SPI_DMA_ADDR_BASE + 0x500) + +#define SPI_DMA_CH_CTL1_OFFSET (0x00) +#define SPI_DMA_CH_XFER_LEN_OFFSET (0x08) +#define SPI_DMA_CH_SAR_LO_OFFSET (0x0C) +#define SPI_DMA_CH_SAR_HI_OFFSET (0x10) +#define SPI_DMA_CH_DAR_LO_OFFSET (0x14) +#define SPI_DMA_CH_DAR_HI_OFFSET (0x18) + +#define SPI_DMA_CH0_DONE_INT BIT(0) +#define SPI_DMA_CH1_DONE_INT BIT(1) +#define SPI_DMA_CH0_ABORT_INT BIT(16) +#define SPI_DMA_CH1_ABORT_INT BIT(17) +#define SPI_DMA_DONE_INT_MASK (SPI_DMA_CH0_DONE_INT | SPI_DMA_CH1_DONE_INT) +#define SPI_DMA_ABORT_INT_MASK (SPI_DMA_CH0_ABORT_INT | SPI_DMA_CH1_ABORT_INT) +#define DMA_CH_CONTROL_LIE BIT(3) +#define DMA_CH_CONTROL_RIE BIT(4) +#define DMA_INTR_EN (DMA_CH_CONTROL_RIE | DMA_CH_CONTROL_LIE) + /* x refers to SPI Host Controller HW instance id in the below macros - 0 or 1 */ #define SPI_MST_CMD_BUF_OFFSET(x) (((x) * SPI_MST1_ADDR_BASE) + 0x00) @@ -50,6 +117,9 @@ #define SPI_MAX_DATA_LEN 320 #define PCI1XXXX_SPI_TIMEOUT (msecs_to_jiffies(100)) +#define SYSLOCK_RETRY_CNT (1000) +#define SPI_DMA_ENGINE_EN (0x1) +#define SPI_DMA_ENGINE_DIS (0x0) #define SPI_INTR BIT(8) #define SPI_FORCE_CE BIT(4) @@ -62,11 +132,21 @@ struct pci1xxxx_spi_internal { u8 hw_inst; - bool spi_xfer_in_progress; + u8 clkdiv; int irq; + int mode; + bool spi_xfer_in_progress; + void *rx_buf; + bool dma_aborted_rd; + u32 bytes_recvd; + u32 tx_sgl_len; + u32 rx_sgl_len; + struct scatterlist *tx_sgl, *rx_sgl; + bool dma_aborted_wr; struct completion spi_xfer_done; struct spi_controller *spi_host; struct pci1xxxx_spi *parent; + struct spi_transfer *xfer; struct { unsigned int dev_sel : 3; unsigned int msi_vector_sel : 1; @@ -76,7 +156,12 @@ struct pci1xxxx_spi_internal { struct pci1xxxx_spi { struct pci_dev *dev; u8 total_hw_instances; + u8 dev_rev; void __iomem *reg_base; + void __iomem *dma_offset_bar; + /* lock to safely access the DMA registers in isr */ + spinlock_t dma_reg_lock; + bool can_dma; struct pci1xxxx_spi_internal *spi_int[] __counted_by(total_hw_instances); }; @@ -106,6 +191,114 @@ static const struct pci_device_id pci1xxxx_spi_pci_id_table[] = { MODULE_DEVICE_TABLE(pci, pci1xxxx_spi_pci_id_table); +static int pci1xxxx_set_sys_lock(struct pci1xxxx_spi *par) +{ + writel(SPI_SYSLOCK, par->reg_base + SPI_SYSLOCK_REG); + return readl(par->reg_base + SPI_SYSLOCK_REG); +} + +static int pci1xxxx_acquire_sys_lock(struct pci1xxxx_spi *par) +{ + u32 regval; + + return readx_poll_timeout(pci1xxxx_set_sys_lock, par, regval, + (regval & SPI_SYSLOCK), 100, + SYSLOCK_RETRY_CNT * 100); +} + +static void pci1xxxx_release_sys_lock(struct pci1xxxx_spi *par) +{ + writel(0x0, par->reg_base + SPI_SYSLOCK_REG); +} + +static int pci1xxxx_check_spi_can_dma(struct pci1xxxx_spi *spi_bus, int irq) +{ + struct pci_dev *pdev = spi_bus->dev; + u32 pf_num; + u32 regval; + int ret; + + /* + * DEV REV Registers is a system register, HW Syslock bit + * should be acquired before accessing the register + */ + ret = pci1xxxx_acquire_sys_lock(spi_bus); + if (ret) { + dev_err(&pdev->dev, "Error failed to acquire syslock\n"); + return ret; + } + + regval = readl(spi_bus->reg_base + DEV_REV_REG); + spi_bus->dev_rev = regval & DEV_REV_MASK; + if (spi_bus->dev_rev >= 0xC0) { + regval = readl(spi_bus->reg_base + + SPI_CONFIG_PERI_ENABLE_REG); + pf_num = regval & SPI_PERI_ENBLE_PF_MASK; + } + + pci1xxxx_release_sys_lock(spi_bus); + + /* + * DMA is supported only from C0 and SPI can use DMA only if + * it is mapped to PF0 + */ + if (spi_bus->dev_rev < 0xC0 || pf_num) + return -EOPNOTSUPP; + + /* + * DMA Supported only with MSI Interrupts + * One of the SPI instance's MSI vector address and data + * is used for DMA Interrupt + */ + if (!irq_get_msi_desc(irq)) { + dev_warn(&pdev->dev, "Error MSI Interrupt not supported, will operate in PIO mode\n"); + return -EOPNOTSUPP; + } + + spi_bus->dma_offset_bar = pcim_iomap(pdev, 2, pci_resource_len(pdev, 2)); + if (!spi_bus->dma_offset_bar) { + dev_warn(&pdev->dev, "Error failed to map dma bar, will operate in PIO mode\n"); + return -EOPNOTSUPP; + } + + if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { + dev_warn(&pdev->dev, "Error failed to set DMA mask, will operate in PIO mode\n"); + pcim_iounmap(pdev, spi_bus->dma_offset_bar); + spi_bus->dma_offset_bar = NULL; + return -EOPNOTSUPP; + } + + return 0; +} + +static int pci1xxxx_spi_dma_init(struct pci1xxxx_spi *spi_bus, int irq) +{ + struct msi_msg msi; + int ret; + + ret = pci1xxxx_check_spi_can_dma(spi_bus, irq); + if (ret) + return ret; + + spin_lock_init(&spi_bus->dma_reg_lock); + get_cached_msi_msg(irq, &msi); + writel(SPI_DMA_ENGINE_EN, spi_bus->dma_offset_bar + SPI_DMA_GLOBAL_WR_ENGINE_EN); + writel(SPI_DMA_ENGINE_EN, spi_bus->dma_offset_bar + SPI_DMA_GLOBAL_RD_ENGINE_EN); + writel(msi.address_hi, spi_bus->dma_offset_bar + SPI_DMA_INTR_IMWR_WDONE_HIGH); + writel(msi.address_hi, spi_bus->dma_offset_bar + SPI_DMA_INTR_IMWR_WABORT_HIGH); + writel(msi.address_hi, spi_bus->dma_offset_bar + SPI_DMA_INTR_IMWR_RDONE_HIGH); + writel(msi.address_hi, spi_bus->dma_offset_bar + SPI_DMA_INTR_IMWR_RABORT_HIGH); + writel(msi.address_lo, spi_bus->dma_offset_bar + SPI_DMA_INTR_IMWR_WDONE_LOW); + writel(msi.address_lo, spi_bus->dma_offset_bar + SPI_DMA_INTR_IMWR_WABORT_LOW); + writel(msi.address_lo, spi_bus->dma_offset_bar + SPI_DMA_INTR_IMWR_RDONE_LOW); + writel(msi.address_lo, spi_bus->dma_offset_bar + SPI_DMA_INTR_IMWR_RABORT_LOW); + writel(msi.data, spi_bus->dma_offset_bar + SPI_DMA_INTR_WR_IMWR_DATA); + writel(msi.data, spi_bus->dma_offset_bar + SPI_DMA_INTR_RD_IMWR_DATA); + dma_set_max_seg_size(&spi_bus->dev->dev, PCI1XXXX_SPI_BUFFER_SIZE); + spi_bus->can_dma = true; + return 0; +} + static void pci1xxxx_spi_set_cs(struct spi_device *spi, bool enable) { struct pci1xxxx_spi_internal *p = spi_controller_get_devdata(spi->controller); @@ -146,12 +339,79 @@ static u8 pci1xxxx_get_clock_div(u32 hz) return val; } -static int pci1xxxx_spi_transfer_one(struct spi_controller *spi_ctlr, - struct spi_device *spi, struct spi_transfer *xfer) +static void pci1xxxx_spi_setup_dma_to_io(struct pci1xxxx_spi_internal *p, + dma_addr_t dma_addr, u32 len) +{ + void __iomem *base; + + if (!p->hw_inst) + base = p->parent->dma_offset_bar + SPI_DMA_CH0_RD_BASE; + else + base = p->parent->dma_offset_bar + SPI_DMA_CH1_RD_BASE; + + writel(DMA_INTR_EN, base + SPI_DMA_CH_CTL1_OFFSET); + writel(len, base + SPI_DMA_CH_XFER_LEN_OFFSET); + writel(lower_32_bits(dma_addr), base + SPI_DMA_CH_SAR_LO_OFFSET); + writel(upper_32_bits(dma_addr), base + SPI_DMA_CH_SAR_HI_OFFSET); + /* Updated SPI Command Registers */ + writel(lower_32_bits(SPI_PERI_ADDR_BASE + SPI_MST_CMD_BUF_OFFSET(p->hw_inst)), + base + SPI_DMA_CH_DAR_LO_OFFSET); + writel(upper_32_bits(SPI_PERI_ADDR_BASE + SPI_MST_CMD_BUF_OFFSET(p->hw_inst)), + base + SPI_DMA_CH_DAR_HI_OFFSET); +} + +static void pci1xxxx_spi_setup_dma_from_io(struct pci1xxxx_spi_internal *p, + dma_addr_t dma_addr, u32 len) +{ + void *base; + + if (!p->hw_inst) + base = p->parent->dma_offset_bar + SPI_DMA_CH0_WR_BASE; + else + base = p->parent->dma_offset_bar + SPI_DMA_CH1_WR_BASE; + + writel(DMA_INTR_EN, base + SPI_DMA_CH_CTL1_OFFSET); + writel(len, base + SPI_DMA_CH_XFER_LEN_OFFSET); + writel(lower_32_bits(dma_addr), base + SPI_DMA_CH_DAR_LO_OFFSET); + writel(upper_32_bits(dma_addr), base + SPI_DMA_CH_DAR_HI_OFFSET); + writel(lower_32_bits(SPI_PERI_ADDR_BASE + SPI_MST_RSP_BUF_OFFSET(p->hw_inst)), + base + SPI_DMA_CH_SAR_LO_OFFSET); + writel(upper_32_bits(SPI_PERI_ADDR_BASE + SPI_MST_RSP_BUF_OFFSET(p->hw_inst)), + base + SPI_DMA_CH_SAR_HI_OFFSET); +} + +static void pci1xxxx_spi_setup(struct pci1xxxx_spi *par, u8 hw_inst, u32 mode, + u8 clkdiv, u32 len) +{ + u32 regval; + + regval = readl(par->reg_base + SPI_MST_CTL_REG_OFFSET(hw_inst)); + regval &= ~(SPI_MST_CTL_MODE_SEL | SPI_MST_CTL_CMD_LEN_MASK | + SPI_MST_CTL_SPEED_MASK); + + if (mode == SPI_MODE_3) + regval |= SPI_MST_CTL_MODE_SEL; + + regval |= FIELD_PREP(SPI_MST_CTL_CMD_LEN_MASK, len); + regval |= FIELD_PREP(SPI_MST_CTL_SPEED_MASK, clkdiv); + writel(regval, par->reg_base + SPI_MST_CTL_REG_OFFSET(hw_inst)); +} + +static void pci1xxxx_start_spi_xfer(struct pci1xxxx_spi_internal *p, u8 hw_inst) +{ + u32 regval; + + regval = readl(p->parent->reg_base + SPI_MST_CTL_REG_OFFSET(hw_inst)); + regval |= SPI_MST_CTL_GO; + writel(regval, p->parent->reg_base + SPI_MST_CTL_REG_OFFSET(hw_inst)); +} + +static int pci1xxxx_spi_transfer_with_io(struct spi_controller *spi_ctlr, + struct spi_device *spi, struct spi_transfer *xfer) { struct pci1xxxx_spi_internal *p = spi_controller_get_devdata(spi_ctlr); - int mode, len, loop_iter, transfer_len; struct pci1xxxx_spi *par = p->parent; + int len, loop_iter, transfer_len; unsigned long bytes_transfered; unsigned long bytes_recvd; unsigned long loop_count; @@ -161,7 +421,7 @@ static int pci1xxxx_spi_transfer_one(struct spi_controller *spi_ctlr, u8 clkdiv; p->spi_xfer_in_progress = true; - mode = spi->mode; + p->bytes_recvd = 0; clkdiv = pci1xxxx_get_clock_div(xfer->speed_hz); tx_buf = xfer->tx_buf; rx_buf = xfer->rx_buf; @@ -186,26 +446,8 @@ static int pci1xxxx_spi_transfer_one(struct spi_controller *spi_ctlr, memcpy_toio(par->reg_base + SPI_MST_CMD_BUF_OFFSET(p->hw_inst), &tx_buf[bytes_transfered], len); bytes_transfered += len; - regval = readl(par->reg_base + - SPI_MST_CTL_REG_OFFSET(p->hw_inst)); - regval &= ~(SPI_MST_CTL_MODE_SEL | SPI_MST_CTL_CMD_LEN_MASK | - SPI_MST_CTL_SPEED_MASK); - - if (mode == SPI_MODE_3) - regval |= SPI_MST_CTL_MODE_SEL; - else - regval &= ~SPI_MST_CTL_MODE_SEL; - - regval |= (clkdiv << 5); - regval &= ~SPI_MST_CTL_CMD_LEN_MASK; - regval |= (len << 8); - writel(regval, par->reg_base + - SPI_MST_CTL_REG_OFFSET(p->hw_inst)); - regval = readl(par->reg_base + - SPI_MST_CTL_REG_OFFSET(p->hw_inst)); - regval |= SPI_MST_CTL_GO; - writel(regval, par->reg_base + - SPI_MST_CTL_REG_OFFSET(p->hw_inst)); + pci1xxxx_spi_setup(par, p->hw_inst, spi->mode, clkdiv, len); + pci1xxxx_start_spi_xfer(p, p->hw_inst); /* Wait for DMA_TERM interrupt */ result = wait_for_completion_timeout(&p->spi_xfer_done, @@ -225,7 +467,113 @@ static int pci1xxxx_spi_transfer_one(struct spi_controller *spi_ctlr, return 0; } -static irqreturn_t pci1xxxx_spi_isr(int irq, void *dev) +static int pci1xxxx_spi_transfer_with_dma(struct spi_controller *spi_ctlr, + struct spi_device *spi, + struct spi_transfer *xfer) +{ + struct pci1xxxx_spi_internal *p = spi_controller_get_devdata(spi_ctlr); + struct pci1xxxx_spi *par = p->parent; + dma_addr_t rx_dma_addr = 0; + dma_addr_t tx_dma_addr = 0; + int ret = 0; + u32 regval; + + p->spi_xfer_in_progress = true; + p->tx_sgl = xfer->tx_sg.sgl; + p->rx_sgl = xfer->rx_sg.sgl; + p->rx_buf = xfer->rx_buf; + regval = readl(par->reg_base + SPI_MST_EVENT_REG_OFFSET(p->hw_inst)); + writel(regval, par->reg_base + SPI_MST_EVENT_REG_OFFSET(p->hw_inst)); + + if (!xfer->tx_buf || !p->tx_sgl) { + ret = -EINVAL; + goto error; + } + p->xfer = xfer; + p->mode = spi->mode; + p->clkdiv = pci1xxxx_get_clock_div(xfer->speed_hz); + p->bytes_recvd = 0; + p->rx_buf = xfer->rx_buf; + regval = readl(par->reg_base + SPI_MST_EVENT_REG_OFFSET(p->hw_inst)); + writel(regval, par->reg_base + SPI_MST_EVENT_REG_OFFSET(p->hw_inst)); + + tx_dma_addr = sg_dma_address(p->tx_sgl); + rx_dma_addr = sg_dma_address(p->rx_sgl); + p->tx_sgl_len = sg_dma_len(p->tx_sgl); + p->rx_sgl_len = sg_dma_len(p->rx_sgl); + pci1xxxx_spi_setup(par, p->hw_inst, p->mode, p->clkdiv, p->tx_sgl_len); + pci1xxxx_spi_setup_dma_to_io(p, (tx_dma_addr), p->tx_sgl_len); + if (rx_dma_addr) + pci1xxxx_spi_setup_dma_from_io(p, rx_dma_addr, p->rx_sgl_len); + writel(p->hw_inst, par->dma_offset_bar + SPI_DMA_RD_DOORBELL_REG); + + reinit_completion(&p->spi_xfer_done); + /* Wait for DMA_TERM interrupt */ + ret = wait_for_completion_timeout(&p->spi_xfer_done, PCI1XXXX_SPI_TIMEOUT); + if (!ret) { + ret = -ETIMEDOUT; + if (p->dma_aborted_rd) { + writel(SPI_DMA_ENGINE_DIS, + par->dma_offset_bar + SPI_DMA_GLOBAL_RD_ENGINE_EN); + /* + * DMA ENGINE reset takes time if any TLP + * completeion in progress, should wait + * till DMA Engine reset is completed. + */ + ret = readl_poll_timeout(par->dma_offset_bar + + SPI_DMA_GLOBAL_RD_ENGINE_EN, regval, + (regval == 0x0), 0, USEC_PER_MSEC); + if (ret) { + ret = -ECANCELED; + goto error; + } + writel(SPI_DMA_ENGINE_EN, + par->dma_offset_bar + SPI_DMA_GLOBAL_RD_ENGINE_EN); + p->dma_aborted_rd = false; + ret = -ECANCELED; + } + if (p->dma_aborted_wr) { + writel(SPI_DMA_ENGINE_DIS, + par->dma_offset_bar + SPI_DMA_GLOBAL_WR_ENGINE_EN); + + /* + * DMA ENGINE reset takes time if any TLP + * completeion in progress, should wait + * till DMA Engine reset is completed. + */ + ret = readl_poll_timeout(par->dma_offset_bar + + SPI_DMA_GLOBAL_WR_ENGINE_EN, regval, + (regval == 0x0), 0, USEC_PER_MSEC); + if (ret) { + ret = -ECANCELED; + goto error; + } + + writel(SPI_DMA_ENGINE_EN, + par->dma_offset_bar + SPI_DMA_GLOBAL_WR_ENGINE_EN); + p->dma_aborted_wr = false; + ret = -ECANCELED; + } + goto error; + } + ret = 0; + +error: + p->spi_xfer_in_progress = false; + + return ret; +} + +static int pci1xxxx_spi_transfer_one(struct spi_controller *spi_ctlr, + struct spi_device *spi, struct spi_transfer *xfer) +{ + if (spi_ctlr->can_dma(spi_ctlr, spi, xfer) && spi_ctlr->cur_msg_mapped) + return pci1xxxx_spi_transfer_with_dma(spi_ctlr, spi, xfer); + else + return pci1xxxx_spi_transfer_with_io(spi_ctlr, spi, xfer); +} + +static irqreturn_t pci1xxxx_spi_isr_io(int irq, void *dev) { struct pci1xxxx_spi_internal *p = dev; irqreturn_t spi_int_fired = IRQ_NONE; @@ -235,15 +583,117 @@ static irqreturn_t pci1xxxx_spi_isr(int irq, void *dev) regval = readl(p->parent->reg_base + SPI_MST_EVENT_REG_OFFSET(p->hw_inst)); if (regval & SPI_INTR) { /* Clear xfer_done */ - complete(&p->spi_xfer_done); + if (p->parent->can_dma && p->rx_buf) + writel(p->hw_inst, p->parent->dma_offset_bar + + SPI_DMA_WR_DOORBELL_REG); + else + complete(&p->parent->spi_int[p->hw_inst]->spi_xfer_done); spi_int_fired = IRQ_HANDLED; } - writel(regval, p->parent->reg_base + SPI_MST_EVENT_REG_OFFSET(p->hw_inst)); - return spi_int_fired; } +static void pci1xxxx_spi_setup_next_dma_transfer(struct pci1xxxx_spi_internal *p) +{ + dma_addr_t tx_dma_addr = 0; + dma_addr_t rx_dma_addr = 0; + u32 prev_len; + + p->tx_sgl = sg_next(p->tx_sgl); + if (p->rx_sgl) + p->rx_sgl = sg_next(p->rx_sgl); + if (!p->tx_sgl) { + /* Clear xfer_done */ + complete(&p->spi_xfer_done); + } else { + tx_dma_addr = sg_dma_address(p->tx_sgl); + prev_len = p->tx_sgl_len; + p->tx_sgl_len = sg_dma_len(p->tx_sgl); + if (prev_len != p->tx_sgl_len) + pci1xxxx_spi_setup(p->parent, + p->hw_inst, p->mode, p->clkdiv, p->tx_sgl_len); + pci1xxxx_spi_setup_dma_to_io(p, tx_dma_addr, p->tx_sgl_len); + if (p->rx_sgl) { + rx_dma_addr = sg_dma_address(p->rx_sgl); + p->rx_sgl_len = sg_dma_len(p->rx_sgl); + pci1xxxx_spi_setup_dma_from_io(p, rx_dma_addr, p->rx_sgl_len); + } + writel(p->hw_inst, p->parent->dma_offset_bar + SPI_DMA_RD_DOORBELL_REG); + } +} + +static irqreturn_t pci1xxxx_spi_isr_dma(int irq, void *dev) +{ + struct pci1xxxx_spi_internal *p = dev; + irqreturn_t spi_int_fired = IRQ_NONE; + unsigned long flags; + u32 regval; + + spin_lock_irqsave(&p->parent->dma_reg_lock, flags); + /* Clear the DMA RD INT and start spi xfer*/ + regval = readl(p->parent->dma_offset_bar + SPI_DMA_INTR_RD_STS); + if (regval & SPI_DMA_DONE_INT_MASK) { + if (regval & SPI_DMA_CH0_DONE_INT) + pci1xxxx_start_spi_xfer(p, SPI0); + if (regval & SPI_DMA_CH1_DONE_INT) + pci1xxxx_start_spi_xfer(p, SPI1); + spi_int_fired = IRQ_HANDLED; + } + if (regval & SPI_DMA_ABORT_INT_MASK) { + p->dma_aborted_rd = true; + spi_int_fired = IRQ_HANDLED; + } + writel(regval, p->parent->dma_offset_bar + SPI_DMA_INTR_RD_CLR); + + /* Clear the DMA WR INT */ + regval = readl(p->parent->dma_offset_bar + SPI_DMA_INTR_WR_STS); + if (regval & SPI_DMA_DONE_INT_MASK) { + if (regval & SPI_DMA_CH0_DONE_INT) + pci1xxxx_spi_setup_next_dma_transfer(p->parent->spi_int[SPI0]); + + if (regval & SPI_DMA_CH1_DONE_INT) + pci1xxxx_spi_setup_next_dma_transfer(p->parent->spi_int[SPI1]); + + spi_int_fired = IRQ_HANDLED; + } + if (regval & SPI_DMA_ABORT_INT_MASK) { + p->dma_aborted_wr = true; + spi_int_fired = IRQ_HANDLED; + } + writel(regval, p->parent->dma_offset_bar + SPI_DMA_INTR_WR_CLR); + spin_unlock_irqrestore(&p->parent->dma_reg_lock, flags); + + /* Clear the SPI GO_BIT Interrupt */ + regval = readl(p->parent->reg_base + SPI_MST_EVENT_REG_OFFSET(p->hw_inst)); + if (regval & SPI_INTR) { + writel(p->hw_inst, p->parent->dma_offset_bar + SPI_DMA_WR_DOORBELL_REG); + spi_int_fired = IRQ_HANDLED; + } + writel(regval, p->parent->reg_base + SPI_MST_EVENT_REG_OFFSET(p->hw_inst)); + return spi_int_fired; +} + +static irqreturn_t pci1xxxx_spi_isr(int irq, void *dev) +{ + struct pci1xxxx_spi_internal *p = dev; + + if (p->spi_host->can_dma(p->spi_host, NULL, p->xfer)) + return pci1xxxx_spi_isr_dma(irq, dev); + else + return pci1xxxx_spi_isr_io(irq, dev); +} + +static bool pci1xxxx_spi_can_dma(struct spi_controller *host, + struct spi_device *spi, + struct spi_transfer *xfer) +{ + struct pci1xxxx_spi_internal *p = spi_controller_get_devdata(host); + struct pci1xxxx_spi *par = p->parent; + + return par->can_dma; +} + static int pci1xxxx_spi_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { u8 hw_inst_cnt, iter, start, only_sec_inst; @@ -324,6 +774,10 @@ static int pci1xxxx_spi_probe(struct pci_dev *pdev, const struct pci_device_id * goto error; } + ret = pci1xxxx_spi_dma_init(spi_bus, spi_sub_ptr->irq); + if (ret && ret != -EOPNOTSUPP) + goto error; + /* This register is only applicable for 1st instance */ regval = readl(spi_bus->reg_base + SPI_PCI_CTRL_REG_OFFSET(0)); if (!only_sec_inst) @@ -360,7 +814,9 @@ static int pci1xxxx_spi_probe(struct pci_dev *pdev, const struct pci_device_id * spi_host->num_chipselect = SPI_CHIP_SEL_COUNT; spi_host->mode_bits = SPI_MODE_0 | SPI_MODE_3 | SPI_RX_DUAL | SPI_TX_DUAL | SPI_LOOP; + spi_host->can_dma = pci1xxxx_spi_can_dma; spi_host->transfer_one = pci1xxxx_spi_transfer_one; + spi_host->set_cs = pci1xxxx_spi_set_cs; spi_host->bits_per_word_mask = SPI_BPW_MASK(8); spi_host->max_speed_hz = PCI1XXXX_SPI_MAX_CLOCK_HZ; diff --git a/drivers/spi/spi-pic32.c b/drivers/spi/spi-pic32.c index f55b38c577e4..709edb70ad7d 100644 --- a/drivers/spi/spi-pic32.c +++ b/drivers/spi/spi-pic32.c @@ -11,13 +11,13 @@ #include #include #include +#include #include #include #include #include #include #include -#include #include #include #include diff --git a/drivers/spi/spi-ppc4xx.c b/drivers/spi/spi-ppc4xx.c index 82d6264841fc..942c3117ab3a 100644 --- a/drivers/spi/spi-ppc4xx.c +++ b/drivers/spi/spi-ppc4xx.c @@ -359,22 +359,22 @@ static int spi_ppc4xx_of_probe(struct platform_device *op) /* Setup the state for the bitbang driver */ bbp = &hw->bitbang; - bbp->master = hw->host; + bbp->ctlr = hw->host; bbp->setup_transfer = spi_ppc4xx_setupxfer; bbp->txrx_bufs = spi_ppc4xx_txrx; bbp->use_dma = 0; - bbp->master->setup = spi_ppc4xx_setup; - bbp->master->cleanup = spi_ppc4xx_cleanup; - bbp->master->bits_per_word_mask = SPI_BPW_MASK(8); - bbp->master->use_gpio_descriptors = true; + bbp->ctlr->setup = spi_ppc4xx_setup; + bbp->ctlr->cleanup = spi_ppc4xx_cleanup; + bbp->ctlr->bits_per_word_mask = SPI_BPW_MASK(8); + bbp->ctlr->use_gpio_descriptors = true; /* * The SPI core will count the number of GPIO descriptors to figure * out the number of chip selects available on the platform. */ - bbp->master->num_chipselect = 0; + bbp->ctlr->num_chipselect = 0; /* the spi->mode bits understood by this driver: */ - bbp->master->mode_bits = + bbp->ctlr->mode_bits = SPI_CPHA | SPI_CPOL | SPI_CS_HIGH | SPI_LSB_FIRST; /* Get the clock for the OPB */ diff --git a/drivers/spi/spi-rockchip.c b/drivers/spi/spi-rockchip.c index 4b9669da2cf3..e1ecd96c7858 100644 --- a/drivers/spi/spi-rockchip.c +++ b/drivers/spi/spi-rockchip.c @@ -160,8 +160,7 @@ */ #define ROCKCHIP_SPI_MAX_TRANLEN 0xffff -/* 2 for native cs, 2 for cs-gpio */ -#define ROCKCHIP_SPI_MAX_CS_NUM 4 +#define ROCKCHIP_SPI_MAX_NATIVE_CS_NUM 2 #define ROCKCHIP_SPI_VER2_TYPE1 0x05EC0002 #define ROCKCHIP_SPI_VER2_TYPE2 0x00110002 @@ -192,8 +191,6 @@ struct rockchip_spi { u8 n_bytes; u8 rsd; - bool cs_asserted[ROCKCHIP_SPI_MAX_CS_NUM]; - bool target_abort; bool cs_inactive; /* spi target tansmition stop when cs inactive */ bool cs_high_supported; /* native CS supports active-high polarity */ @@ -245,10 +242,6 @@ static void rockchip_spi_set_cs(struct spi_device *spi, bool enable) struct rockchip_spi *rs = spi_controller_get_devdata(ctlr); bool cs_asserted = spi->mode & SPI_CS_HIGH ? enable : !enable; - /* Return immediately for no-op */ - if (cs_asserted == rs->cs_asserted[spi_get_chipselect(spi, 0)]) - return; - if (cs_asserted) { /* Keep things powered as long as CS is asserted */ pm_runtime_get_sync(rs->dev); @@ -268,8 +261,6 @@ static void rockchip_spi_set_cs(struct spi_device *spi, bool enable) /* Drop reference from when we first asserted CS */ pm_runtime_put(rs->dev); } - - rs->cs_asserted[spi_get_chipselect(spi, 0)] = cs_asserted; } static void rockchip_spi_handle_err(struct spi_controller *ctlr, @@ -847,7 +838,7 @@ static int rockchip_spi_probe(struct platform_device *pdev) ctlr->target_abort = rockchip_spi_target_abort; } else { ctlr->flags = SPI_CONTROLLER_GPIO_SS; - ctlr->max_native_cs = ROCKCHIP_SPI_MAX_CS_NUM; + ctlr->max_native_cs = ROCKCHIP_SPI_MAX_NATIVE_CS_NUM; /* * rk spi0 has two native cs, spi1..5 one cs only * if num-cs is missing in the dts, default to 1 diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 0e48ffd499b9..9fcbe040cb2f 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -3,19 +3,22 @@ // Copyright (c) 2009 Samsung Electronics Co., Ltd. // Jaswinder Singh -#include -#include -#include -#include +#include +#include #include +#include #include #include +#include +#include +#include +#include +#include +#include #include #include #include -#include - -#include +#include #define MAX_SPI_PORTS 12 #define S3C64XX_SPI_QUIRK_CS_AUTO (1 << 1) @@ -76,6 +79,9 @@ #define S3C64XX_SPI_INT_RX_FIFORDY_EN (1<<1) #define S3C64XX_SPI_INT_TX_FIFORDY_EN (1<<0) +#define S3C64XX_SPI_ST_RX_FIFO_RDY_V2 GENMASK(23, 15) +#define S3C64XX_SPI_ST_TX_FIFO_RDY_V2 GENMASK(14, 6) +#define S3C64XX_SPI_ST_TX_FIFO_LVL_SHIFT 6 #define S3C64XX_SPI_ST_RX_OVERRUN_ERR (1<<5) #define S3C64XX_SPI_ST_RX_UNDERRUN_ERR (1<<4) #define S3C64XX_SPI_ST_TX_OVERRUN_ERR (1<<3) @@ -106,15 +112,15 @@ #define FIFO_LVL_MASK(i) ((i)->port_conf->fifo_lvl_mask[i->port_id]) #define S3C64XX_SPI_ST_TX_DONE(v, i) (((v) & \ (1 << (i)->port_conf->tx_st_done)) ? 1 : 0) -#define TX_FIFO_LVL(v, i) (((v) >> 6) & FIFO_LVL_MASK(i)) -#define RX_FIFO_LVL(v, i) (((v) >> (i)->port_conf->rx_lvl_offset) & \ - FIFO_LVL_MASK(i)) +#define TX_FIFO_LVL(v, sdd) (((v) & (sdd)->tx_fifomask) >> \ + __ffs((sdd)->tx_fifomask)) +#define RX_FIFO_LVL(v, sdd) (((v) & (sdd)->rx_fifomask) >> \ + __ffs((sdd)->rx_fifomask)) +#define FIFO_DEPTH(i) ((FIFO_LVL_MASK(i) >> 1) + 1) #define S3C64XX_SPI_MAX_TRAILCNT 0x3ff #define S3C64XX_SPI_TRAILCNT_OFF 19 -#define S3C64XX_SPI_TRAILCNT S3C64XX_SPI_MAX_TRAILCNT - #define S3C64XX_SPI_POLLING_SIZE 32 #define msecs_to_loops(t) (loops_per_jiffy / 1000 * HZ * t) @@ -131,8 +137,13 @@ struct s3c64xx_spi_dma_data { /** * struct s3c64xx_spi_port_config - SPI Controller hardware info - * @fifo_lvl_mask: Bit-mask for {TX|RX}_FIFO_LVL bits in SPI_STATUS register. - * @rx_lvl_offset: Bit offset of RX_FIFO_LVL bits in SPI_STATUS regiter. + * @fifo_lvl_mask: [DEPRECATED] use @{rx, tx}_fifomask instead. + * @rx_lvl_offset: [DEPRECATED] use @{rx,tx}_fifomask instead. + * @fifo_depth: depth of the FIFO. + * @rx_fifomask: SPI_STATUS.RX_FIFO_LVL mask. Shifted mask defining the field's + * length and position. + * @tx_fifomask: SPI_STATUS.TX_FIFO_LVL mask. Shifted mask defining the field's + * length and position. * @tx_st_done: Bit offset of TX_DONE bit in SPI_STATUS regiter. * @clk_div: Internal clock divider * @quirks: Bitmask of known quirks @@ -141,6 +152,7 @@ struct s3c64xx_spi_dma_data { * prescaler unit. * @clk_ioclk: True if clock is present on this device * @has_loopback: True if loopback mode can be supported + * @use_32bit_io: True if the SoC allows only 32-bit register accesses. * * The Samsung s3c64xx SPI controller are used on various Samsung SoC's but * differ in some aspects such as the size of the fifo and spi bus clock @@ -150,6 +162,9 @@ struct s3c64xx_spi_dma_data { struct s3c64xx_spi_port_config { int fifo_lvl_mask[MAX_SPI_PORTS]; int rx_lvl_offset; + unsigned int fifo_depth; + u32 rx_fifomask; + u32 tx_fifomask; int tx_st_done; int quirks; int clk_div; @@ -157,6 +172,7 @@ struct s3c64xx_spi_port_config { bool clk_from_cmu; bool clk_ioclk; bool has_loopback; + bool use_32bit_io; }; /** @@ -177,8 +193,13 @@ struct s3c64xx_spi_port_config { * @cur_speed: Current clock speed * @rx_dma: Local receive DMA data (e.g. chan and direction) * @tx_dma: Local transmit DMA data (e.g. chan and direction) - * @port_conf: Local SPI port configuartion data - * @port_id: Port identification number + * @port_conf: Local SPI port configuration data + * @port_id: [DEPRECATED] use @{rx,tx}_fifomask instead. + * @fifo_depth: depth of the FIFO. + * @rx_fifomask: SPI_STATUS.RX_FIFO_LVL mask. Shifted mask defining the field's + * length and position. + * @tx_fifomask: SPI_STATUS.TX_FIFO_LVL mask. Shifted mask defining the field's + * length and position. */ struct s3c64xx_spi_driver_data { void __iomem *regs; @@ -198,6 +219,9 @@ struct s3c64xx_spi_driver_data { struct s3c64xx_spi_dma_data tx_dma; const struct s3c64xx_spi_port_config *port_conf; unsigned int port_id; + unsigned int fifo_depth; + u32 rx_fifomask; + u32 tx_fifomask; }; static void s3c64xx_flush_fifo(struct s3c64xx_spi_driver_data *sdd) @@ -276,8 +300,8 @@ static void s3c64xx_spi_dmacb(void *data) spin_unlock_irqrestore(&sdd->lock, flags); } -static int prepare_dma(struct s3c64xx_spi_dma_data *dma, - struct sg_table *sgt) +static int s3c64xx_prepare_dma(struct s3c64xx_spi_dma_data *dma, + struct sg_table *sgt) { struct s3c64xx_spi_driver_data *sdd; struct dma_slave_config config; @@ -289,20 +313,20 @@ static int prepare_dma(struct s3c64xx_spi_dma_data *dma, if (dma->direction == DMA_DEV_TO_MEM) { sdd = container_of((void *)dma, struct s3c64xx_spi_driver_data, rx_dma); - config.direction = dma->direction; config.src_addr = sdd->sfr_start + S3C64XX_SPI_RX_DATA; config.src_addr_width = sdd->cur_bpw / 8; config.src_maxburst = 1; - dmaengine_slave_config(dma->ch, &config); } else { sdd = container_of((void *)dma, struct s3c64xx_spi_driver_data, tx_dma); - config.direction = dma->direction; config.dst_addr = sdd->sfr_start + S3C64XX_SPI_TX_DATA; config.dst_addr_width = sdd->cur_bpw / 8; config.dst_maxburst = 1; - dmaengine_slave_config(dma->ch, &config); } + config.direction = dma->direction; + ret = dmaengine_slave_config(dma->ch, &config); + if (ret) + return ret; desc = dmaengine_prep_slave_sg(dma->ch, sgt->sgl, sgt->nents, dma->direction, DMA_PREP_INTERRUPT); @@ -319,7 +343,7 @@ static int prepare_dma(struct s3c64xx_spi_dma_data *dma, ret = dma_submit_error(dma->cookie); if (ret) { dev_err(&sdd->pdev->dev, "DMA submission failed"); - return -EIO; + return ret; } dma_async_issue_pending(dma->ch); @@ -405,12 +429,60 @@ static bool s3c64xx_spi_can_dma(struct spi_controller *host, { struct s3c64xx_spi_driver_data *sdd = spi_controller_get_devdata(host); - if (sdd->rx_dma.ch && sdd->tx_dma.ch) { - return xfer->len > (FIFO_LVL_MASK(sdd) >> 1) + 1; - } else { - return false; - } + if (sdd->rx_dma.ch && sdd->tx_dma.ch) + return xfer->len > sdd->fifo_depth; + return false; +} + +static void s3c64xx_iowrite8_32_rep(volatile void __iomem *addr, + const void *buffer, unsigned int count) +{ + if (count) { + const u8 *buf = buffer; + + do { + __raw_writel(*buf++, addr); + } while (--count); + } +} + +static void s3c64xx_iowrite16_32_rep(volatile void __iomem *addr, + const void *buffer, unsigned int count) +{ + if (count) { + const u16 *buf = buffer; + + do { + __raw_writel(*buf++, addr); + } while (--count); + } +} + +static void s3c64xx_iowrite_rep(const struct s3c64xx_spi_driver_data *sdd, + struct spi_transfer *xfer) +{ + void __iomem *addr = sdd->regs + S3C64XX_SPI_TX_DATA; + const void *buf = xfer->tx_buf; + unsigned int len = xfer->len; + + switch (sdd->cur_bpw) { + case 32: + iowrite32_rep(addr, buf, len / 4); + break; + case 16: + if (sdd->port_conf->use_32bit_io) + s3c64xx_iowrite16_32_rep(addr, buf, len / 2); + else + iowrite16_rep(addr, buf, len / 2); + break; + default: + if (sdd->port_conf->use_32bit_io) + s3c64xx_iowrite8_32_rep(addr, buf, len); + else + iowrite8_rep(addr, buf, len); + break; + } } static int s3c64xx_enable_datapath(struct s3c64xx_spi_driver_data *sdd, @@ -444,22 +516,9 @@ static int s3c64xx_enable_datapath(struct s3c64xx_spi_driver_data *sdd, chcfg |= S3C64XX_SPI_CH_TXCH_ON; if (dma_mode) { modecfg |= S3C64XX_SPI_MODE_TXDMA_ON; - ret = prepare_dma(&sdd->tx_dma, &xfer->tx_sg); + ret = s3c64xx_prepare_dma(&sdd->tx_dma, &xfer->tx_sg); } else { - switch (sdd->cur_bpw) { - case 32: - iowrite32_rep(regs + S3C64XX_SPI_TX_DATA, - xfer->tx_buf, xfer->len / 4); - break; - case 16: - iowrite16_rep(regs + S3C64XX_SPI_TX_DATA, - xfer->tx_buf, xfer->len / 2); - break; - default: - iowrite8_rep(regs + S3C64XX_SPI_TX_DATA, - xfer->tx_buf, xfer->len); - break; - } + s3c64xx_iowrite_rep(sdd, xfer); } } @@ -476,7 +535,7 @@ static int s3c64xx_enable_datapath(struct s3c64xx_spi_driver_data *sdd, writel(((xfer->len * 8 / sdd->cur_bpw) & 0xffff) | S3C64XX_SPI_PACKET_CNT_EN, regs + S3C64XX_SPI_PACKET_CNT); - ret = prepare_dma(&sdd->rx_dma, &xfer->rx_sg); + ret = s3c64xx_prepare_dma(&sdd->rx_dma, &xfer->rx_sg); } } @@ -495,9 +554,7 @@ static u32 s3c64xx_spi_wait_for_timeout(struct s3c64xx_spi_driver_data *sdd, void __iomem *regs = sdd->regs; unsigned long val = 1; u32 status; - - /* max fifo depth available */ - u32 max_fifo = (FIFO_LVL_MASK(sdd) >> 1) + 1; + u32 max_fifo = sdd->fifo_depth; if (timeout_ms) val = msecs_to_loops(timeout_ms); @@ -528,7 +585,7 @@ static int s3c64xx_wait_for_dma(struct s3c64xx_spi_driver_data *sdd, /* * If the previous xfer was completed within timeout, then - * proceed further else return -EIO. + * proceed further else return -ETIMEDOUT. * DmaTx returns after simply writing data in the FIFO, * w/o waiting for real transmission on the bus to finish. * DmaRx returns only after Dma read data from FIFO which @@ -549,7 +606,7 @@ static int s3c64xx_wait_for_dma(struct s3c64xx_spi_driver_data *sdd, /* If timed out while checking rx/tx status return error */ if (!val) - return -EIO; + return -ETIMEDOUT; return 0; } @@ -579,7 +636,7 @@ static int s3c64xx_wait_for_pio(struct s3c64xx_spi_driver_data *sdd, if (use_irq) { val = msecs_to_jiffies(ms); if (!wait_for_completion_timeout(&sdd->xfer_completion, val)) - return -EIO; + return -ETIMEDOUT; } val = msecs_to_loops(ms); @@ -604,7 +661,7 @@ static int s3c64xx_wait_for_pio(struct s3c64xx_spi_driver_data *sdd, * For any size less than the fifo size the below code is * executed atleast once. */ - loops = xfer->len / ((FIFO_LVL_MASK(sdd) >> 1) + 1); + loops = xfer->len / sdd->fifo_depth; buf = xfer->rx_buf; do { /* wait for data to be received in the fifo */ @@ -741,7 +798,7 @@ static int s3c64xx_spi_transfer_one(struct spi_controller *host, struct spi_transfer *xfer) { struct s3c64xx_spi_driver_data *sdd = spi_controller_get_devdata(host); - const unsigned int fifo_len = (FIFO_LVL_MASK(sdd) >> 1) + 1; + const unsigned int fifo_len = sdd->fifo_depth; const void *tx_buf = NULL; void *rx_buf = NULL; int target_len = 0, origin_len = 0; @@ -1093,8 +1150,7 @@ static void s3c64xx_spi_hwinit(struct s3c64xx_spi_driver_data *sdd) val = readl(regs + S3C64XX_SPI_MODE_CFG); val &= ~S3C64XX_SPI_MODE_4BURST; - val &= ~(S3C64XX_SPI_MAX_TRAILCNT << S3C64XX_SPI_TRAILCNT_OFF); - val |= (S3C64XX_SPI_TRAILCNT << S3C64XX_SPI_TRAILCNT_OFF); + val |= (S3C64XX_SPI_MAX_TRAILCNT << S3C64XX_SPI_TRAILCNT_OFF); writel(val, regs + S3C64XX_SPI_MODE_CFG); s3c64xx_flush_fifo(sdd); @@ -1111,14 +1167,14 @@ static struct s3c64xx_spi_info *s3c64xx_spi_parse_dt(struct device *dev) return ERR_PTR(-ENOMEM); if (of_property_read_u32(dev->of_node, "samsung,spi-src-clk", &temp)) { - dev_warn(dev, "spi bus clock parent not specified, using clock at index 0 as parent\n"); + dev_dbg(dev, "spi bus clock parent not specified, using clock at index 0 as parent\n"); sci->src_clk_nr = 0; } else { sci->src_clk_nr = temp; } if (of_property_read_u32(dev->of_node, "num-cs", &temp)) { - dev_warn(dev, "number of chip select lines not specified, assuming 1 chip select line\n"); + dev_dbg(dev, "number of chip select lines not specified, assuming 1 chip select line\n"); sci->num_cs = 1; } else { sci->num_cs = temp; @@ -1146,6 +1202,48 @@ static inline const struct s3c64xx_spi_port_config *s3c64xx_spi_get_port_config( return (const struct s3c64xx_spi_port_config *)platform_get_device_id(pdev)->driver_data; } +static int s3c64xx_spi_set_port_id(struct platform_device *pdev, + struct s3c64xx_spi_driver_data *sdd) +{ + const struct s3c64xx_spi_port_config *port_conf = sdd->port_conf; + int ret; + + if (port_conf->rx_fifomask && port_conf->tx_fifomask) + return 0; + + if (pdev->dev.of_node) { + ret = of_alias_get_id(pdev->dev.of_node, "spi"); + if (ret < 0) + return dev_err_probe(&pdev->dev, ret, + "Failed to get alias id\n"); + sdd->port_id = ret; + } else { + if (pdev->id < 0) + return dev_err_probe(&pdev->dev, -EINVAL, + "Negative platform ID is not allowed\n"); + sdd->port_id = pdev->id; + } + + return 0; +} + +static void s3c64xx_spi_set_fifomask(struct s3c64xx_spi_driver_data *sdd) +{ + const struct s3c64xx_spi_port_config *port_conf = sdd->port_conf; + + if (port_conf->rx_fifomask) + sdd->rx_fifomask = port_conf->rx_fifomask; + else + sdd->rx_fifomask = FIFO_LVL_MASK(sdd) << + port_conf->rx_lvl_offset; + + if (port_conf->tx_fifomask) + sdd->tx_fifomask = port_conf->tx_fifomask; + else + sdd->tx_fifomask = FIFO_LVL_MASK(sdd) << + S3C64XX_SPI_ST_TX_FIFO_LVL_SHIFT; +} + static int s3c64xx_spi_probe(struct platform_device *pdev) { struct resource *mem_res; @@ -1181,15 +1279,18 @@ static int s3c64xx_spi_probe(struct platform_device *pdev) sdd->host = host; sdd->cntrlr_info = sci; sdd->pdev = pdev; - if (pdev->dev.of_node) { - ret = of_alias_get_id(pdev->dev.of_node, "spi"); - if (ret < 0) - return dev_err_probe(&pdev->dev, ret, - "Failed to get alias id\n"); - sdd->port_id = ret; - } else { - sdd->port_id = pdev->id; - } + + ret = s3c64xx_spi_set_port_id(pdev, sdd); + if (ret) + return ret; + + if (sdd->port_conf->fifo_depth) + sdd->fifo_depth = sdd->port_conf->fifo_depth; + else if (of_property_read_u32(pdev->dev.of_node, "fifo-depth", + &sdd->fifo_depth)) + sdd->fifo_depth = FIFO_DEPTH(sdd); + + s3c64xx_spi_set_fifomask(sdd); sdd->cur_bpw = 8; @@ -1197,7 +1298,7 @@ static int s3c64xx_spi_probe(struct platform_device *pdev) sdd->rx_dma.direction = DMA_DEV_TO_MEM; host->dev.of_node = pdev->dev.of_node; - host->bus_num = sdd->port_id; + host->bus_num = -1; host->setup = s3c64xx_spi_setup; host->cleanup = s3c64xx_spi_cleanup; host->prepare_transfer_hardware = s3c64xx_spi_prepare_transfer; @@ -1278,9 +1379,9 @@ static int s3c64xx_spi_probe(struct platform_device *pdev) } dev_dbg(&pdev->dev, "Samsung SoC SPI Driver loaded for Bus SPI-%d with %d Targets attached\n", - sdd->port_id, host->num_chipselect); + host->bus_num, host->num_chipselect); dev_dbg(&pdev->dev, "\tIOmem=[%pR]\tFIFO %dbytes\n", - mem_res, (FIFO_LVL_MASK(sdd) >> 1) + 1); + mem_res, sdd->fifo_depth); pm_runtime_mark_last_busy(&pdev->dev); pm_runtime_put_autosuspend(&pdev->dev); @@ -1319,8 +1420,9 @@ static int s3c64xx_spi_suspend(struct device *dev) { struct spi_controller *host = dev_get_drvdata(dev); struct s3c64xx_spi_driver_data *sdd = spi_controller_get_devdata(host); + int ret; - int ret = spi_controller_suspend(host); + ret = spi_controller_suspend(host); if (ret) return ret; @@ -1408,7 +1510,9 @@ static const struct dev_pm_ops s3c64xx_spi_pm = { }; static const struct s3c64xx_spi_port_config s3c2443_spi_port_config = { + /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ .fifo_lvl_mask = { 0x7f }, + /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ .rx_lvl_offset = 13, .tx_st_done = 21, .clk_div = 2, @@ -1416,14 +1520,18 @@ static const struct s3c64xx_spi_port_config s3c2443_spi_port_config = { }; static const struct s3c64xx_spi_port_config s3c6410_spi_port_config = { + /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ .fifo_lvl_mask = { 0x7f, 0x7F }, + /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ .rx_lvl_offset = 13, .tx_st_done = 21, .clk_div = 2, }; static const struct s3c64xx_spi_port_config s5pv210_spi_port_config = { + /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ .fifo_lvl_mask = { 0x1ff, 0x7F }, + /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ .rx_lvl_offset = 15, .tx_st_done = 25, .clk_div = 2, @@ -1431,7 +1539,9 @@ static const struct s3c64xx_spi_port_config s5pv210_spi_port_config = { }; static const struct s3c64xx_spi_port_config exynos4_spi_port_config = { + /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ .fifo_lvl_mask = { 0x1ff, 0x7F, 0x7F }, + /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ .rx_lvl_offset = 15, .tx_st_done = 25, .clk_div = 2, @@ -1441,7 +1551,9 @@ static const struct s3c64xx_spi_port_config exynos4_spi_port_config = { }; static const struct s3c64xx_spi_port_config exynos7_spi_port_config = { + /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ .fifo_lvl_mask = { 0x1ff, 0x7F, 0x7F, 0x7F, 0x7F, 0x1ff}, + /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ .rx_lvl_offset = 15, .tx_st_done = 25, .clk_div = 2, @@ -1451,7 +1563,9 @@ static const struct s3c64xx_spi_port_config exynos7_spi_port_config = { }; static const struct s3c64xx_spi_port_config exynos5433_spi_port_config = { + /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ .fifo_lvl_mask = { 0x1ff, 0x7f, 0x7f, 0x7f, 0x7f, 0x1ff}, + /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ .rx_lvl_offset = 15, .tx_st_done = 25, .clk_div = 2, @@ -1461,9 +1575,23 @@ static const struct s3c64xx_spi_port_config exynos5433_spi_port_config = { .quirks = S3C64XX_SPI_QUIRK_CS_AUTO, }; +static const struct s3c64xx_spi_port_config exynos850_spi_port_config = { + .fifo_depth = 64, + .rx_fifomask = S3C64XX_SPI_ST_RX_FIFO_RDY_V2, + .tx_fifomask = S3C64XX_SPI_ST_TX_FIFO_RDY_V2, + .tx_st_done = 25, + .clk_div = 4, + .high_speed = true, + .clk_from_cmu = true, + .has_loopback = true, + .quirks = S3C64XX_SPI_QUIRK_CS_AUTO, +}; + static const struct s3c64xx_spi_port_config exynosautov9_spi_port_config = { + /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ .fifo_lvl_mask = { 0x1ff, 0x1ff, 0x7f, 0x7f, 0x7f, 0x7f, 0x1ff, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f}, + /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ .rx_lvl_offset = 15, .tx_st_done = 25, .clk_div = 4, @@ -1475,7 +1603,9 @@ static const struct s3c64xx_spi_port_config exynosautov9_spi_port_config = { }; static const struct s3c64xx_spi_port_config fsd_spi_port_config = { + /* fifo_lvl_mask is deprecated. Use {rx, tx}_fifomask instead. */ .fifo_lvl_mask = { 0x7f, 0x7f, 0x7f, 0x7f, 0x7f}, + /* rx_lvl_offset is deprecated. Use {rx, tx}_fifomask instead. */ .rx_lvl_offset = 15, .tx_st_done = 25, .clk_div = 2, @@ -1485,6 +1615,19 @@ static const struct s3c64xx_spi_port_config fsd_spi_port_config = { .quirks = S3C64XX_SPI_QUIRK_CS_AUTO, }; +static const struct s3c64xx_spi_port_config gs101_spi_port_config = { + .fifo_depth = 64, + .rx_fifomask = S3C64XX_SPI_ST_RX_FIFO_RDY_V2, + .tx_fifomask = S3C64XX_SPI_ST_TX_FIFO_RDY_V2, + .tx_st_done = 25, + .clk_div = 4, + .high_speed = true, + .clk_from_cmu = true, + .has_loopback = true, + .use_32bit_io = true, + .quirks = S3C64XX_SPI_QUIRK_CS_AUTO, +}; + static const struct platform_device_id s3c64xx_spi_driver_ids[] = { { .name = "s3c2443-spi", @@ -1497,29 +1640,35 @@ static const struct platform_device_id s3c64xx_spi_driver_ids[] = { }; static const struct of_device_id s3c64xx_spi_dt_match[] = { + { .compatible = "google,gs101-spi", + .data = &gs101_spi_port_config, + }, { .compatible = "samsung,s3c2443-spi", - .data = (void *)&s3c2443_spi_port_config, + .data = &s3c2443_spi_port_config, }, { .compatible = "samsung,s3c6410-spi", - .data = (void *)&s3c6410_spi_port_config, + .data = &s3c6410_spi_port_config, }, { .compatible = "samsung,s5pv210-spi", - .data = (void *)&s5pv210_spi_port_config, + .data = &s5pv210_spi_port_config, }, { .compatible = "samsung,exynos4210-spi", - .data = (void *)&exynos4_spi_port_config, + .data = &exynos4_spi_port_config, }, { .compatible = "samsung,exynos7-spi", - .data = (void *)&exynos7_spi_port_config, + .data = &exynos7_spi_port_config, }, { .compatible = "samsung,exynos5433-spi", - .data = (void *)&exynos5433_spi_port_config, + .data = &exynos5433_spi_port_config, + }, + { .compatible = "samsung,exynos850-spi", + .data = &exynos850_spi_port_config, }, { .compatible = "samsung,exynosautov9-spi", - .data = (void *)&exynosautov9_spi_port_config, + .data = &exynosautov9_spi_port_config, }, { .compatible = "tesla,fsd-spi", - .data = (void *)&fsd_spi_port_config, + .data = &fsd_spi_port_config, }, { }, }; diff --git a/drivers/spi/spi-sh-sci.c b/drivers/spi/spi-sh-sci.c index 148d615d2f38..3d560b154ad3 100644 --- a/drivers/spi/spi-sh-sci.c +++ b/drivers/spi/spi-sh-sci.c @@ -136,9 +136,9 @@ static int sh_sci_spi_probe(struct platform_device *dev) } /* setup spi bitbang adaptor */ - sp->bitbang.master = host; - sp->bitbang.master->bus_num = sp->info->bus_num; - sp->bitbang.master->num_chipselect = sp->info->num_chipselect; + sp->bitbang.ctlr = host; + sp->bitbang.ctlr->bus_num = sp->info->bus_num; + sp->bitbang.ctlr->num_chipselect = sp->info->num_chipselect; sp->bitbang.chipselect = sh_sci_spi_chipselect; sp->bitbang.txrx_word[SPI_MODE_0] = sh_sci_spi_txrx_mode0; @@ -166,7 +166,7 @@ static int sh_sci_spi_probe(struct platform_device *dev) setbits(sp, PIN_INIT, 0); iounmap(sp->membase); err1: - spi_controller_put(sp->bitbang.master); + spi_controller_put(sp->bitbang.ctlr); err0: return ret; } @@ -178,7 +178,7 @@ static void sh_sci_spi_remove(struct platform_device *dev) spi_bitbang_stop(&sp->bitbang); setbits(sp, PIN_INIT, 0); iounmap(sp->membase); - spi_controller_put(sp->bitbang.master); + spi_controller_put(sp->bitbang.ctlr); } static struct platform_driver sh_sci_spi_drv = { diff --git a/drivers/spi/spi-slave-mt27xx.c b/drivers/spi/spi-slave-mt27xx.c index 6d6772974783..f1ddf4c099a3 100644 --- a/drivers/spi/spi-slave-mt27xx.c +++ b/drivers/spi/spi-slave-mt27xx.c @@ -297,7 +297,7 @@ static int mtk_spi_slave_transfer_one(struct spi_controller *ctlr, static int mtk_spi_slave_setup(struct spi_device *spi) { - struct mtk_spi_slave *mdata = spi_controller_get_devdata(spi->master); + struct mtk_spi_slave *mdata = spi_controller_get_devdata(spi->controller); u32 reg_val; reg_val = DMA_DONE_EN | DATA_DONE_EN | diff --git a/drivers/spi/spi-stm32-qspi.c b/drivers/spi/spi-stm32-qspi.c index 385832030459..f1e922fd362a 100644 --- a/drivers/spi/spi-stm32-qspi.c +++ b/drivers/spi/spi-stm32-qspi.c @@ -8,13 +8,13 @@ #include #include #include +#include #include #include #include #include #include #include -#include #include #include #include diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c index e61302ef3c21..e4e7ddb7524a 100644 --- a/drivers/spi/spi-stm32.c +++ b/drivers/spi/spi-stm32.c @@ -1118,6 +1118,21 @@ static irqreturn_t stm32h7_spi_irq_thread(int irq, void *dev_id) return IRQ_HANDLED; } +static int stm32_spi_optimize_message(struct spi_message *msg) +{ + struct spi_controller *ctrl = msg->spi->controller; + struct stm32_spi *spi = spi_controller_get_devdata(ctrl); + + /* On STM32H7, messages should not exceed a maximum size set + * later via the set_number_of_data function. In order to + * ensure that, split large messages into several messages + */ + if (spi->cfg->set_number_of_data) + return spi_split_transfers_maxwords(ctrl, msg, spi->t_size_max); + + return 0; +} + /** * stm32_spi_prepare_msg - set up the controller to transfer a single message * @ctrl: controller interface @@ -1163,20 +1178,6 @@ static int stm32_spi_prepare_msg(struct spi_controller *ctrl, !!(spi_dev->mode & SPI_LSB_FIRST), !!(spi_dev->mode & SPI_CS_HIGH)); - /* On STM32H7, messages should not exceed a maximum size setted - * afterward via the set_number_of_data function. In order to - * ensure that, split large messages into several messages - */ - if (spi->cfg->set_number_of_data) { - int ret; - - ret = spi_split_transfers_maxwords(ctrl, msg, - spi->t_size_max, - GFP_KERNEL | GFP_DMA); - if (ret) - return ret; - } - spin_lock_irqsave(&spi->lock, flags); /* CPOL, CPHA and LSB FIRST bits have common register */ @@ -2182,6 +2183,7 @@ static int stm32_spi_probe(struct platform_device *pdev) ctrl->max_speed_hz = spi->clk_rate / spi->cfg->baud_rate_div_min; ctrl->min_speed_hz = spi->clk_rate / spi->cfg->baud_rate_div_max; ctrl->use_gpio_descriptors = true; + ctrl->optimize_message = stm32_spi_optimize_message; ctrl->prepare_message = stm32_spi_prepare_msg; ctrl->transfer_one = stm32_spi_transfer_one; ctrl->unprepare_message = stm32_spi_unprepare_msg; diff --git a/drivers/spi/spi-xilinx.c b/drivers/spi/spi-xilinx.c index 12355957be97..7795328427a6 100644 --- a/drivers/spi/spi-xilinx.c +++ b/drivers/spi/spi-xilinx.c @@ -434,7 +434,7 @@ static int xilinx_spi_probe(struct platform_device *pdev) xspi = spi_controller_get_devdata(host); xspi->cs_inactive = 0xffffffff; - xspi->bitbang.master = host; + xspi->bitbang.ctlr = host; xspi->bitbang.chipselect = xilinx_spi_chipselect; xspi->bitbang.setup_transfer = xilinx_spi_setup_transfer; xspi->bitbang.txrx_bufs = xilinx_spi_txrx_bufs; @@ -516,7 +516,7 @@ static void xilinx_spi_remove(struct platform_device *pdev) /* Disable the global IPIF interrupt */ xspi->write_fn(0, regs_base + XIPIF_V123B_DGIER_OFFSET); - spi_controller_put(xspi->bitbang.master); + spi_controller_put(xspi->bitbang.ctlr); } /* work with hotplug and coldplug */ diff --git a/drivers/spi/spi-xtensa-xtfpga.c b/drivers/spi/spi-xtensa-xtfpga.c index 3c7721894376..3c2cda315397 100644 --- a/drivers/spi/spi-xtensa-xtfpga.c +++ b/drivers/spi/spi-xtensa-xtfpga.c @@ -93,7 +93,7 @@ static int xtfpga_spi_probe(struct platform_device *pdev) host->dev.of_node = pdev->dev.of_node; xspi = spi_controller_get_devdata(host); - xspi->bitbang.master = host; + xspi->bitbang.ctlr = host; xspi->bitbang.chipselect = xtfpga_spi_chipselect; xspi->bitbang.txrx_word[SPI_MODE_0] = xtfpga_spi_txrx_word; xspi->regs = devm_platform_ioremap_resource(pdev, 0); diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index f2170f4b5077..f18738ae95f8 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -459,7 +459,7 @@ static void spi_shutdown(struct device *dev) } } -struct bus_type spi_bus_type = { +const struct bus_type spi_bus_type = { .name = "spi", .dev_groups = spi_dev_groups, .match = spi_match_device, @@ -584,7 +584,7 @@ struct spi_device *spi_alloc_device(struct spi_controller *ctlr) return NULL; } - spi->master = spi->controller = ctlr; + spi->controller = ctlr; spi->dev.parent = &ctlr->dev; spi->dev.bus = &spi_bus_type; spi->dev.release = spidev_release; @@ -608,23 +608,51 @@ static void spi_dev_set_name(struct spi_device *spi) spi_get_chipselect(spi, 0)); } +/* + * Zero(0) is a valid physical CS value and can be located at any + * logical CS in the spi->chip_select[]. If all the physical CS + * are initialized to 0 then It would be difficult to differentiate + * between a valid physical CS 0 & an unused logical CS whose physical + * CS can be 0. As a solution to this issue initialize all the CS to -1. + * Now all the unused logical CS will have -1 physical CS value & can be + * ignored while performing physical CS validity checks. + */ +#define SPI_INVALID_CS ((s8)-1) + +static inline bool is_valid_cs(s8 chip_select) +{ + return chip_select != SPI_INVALID_CS; +} + +static inline int spi_dev_check_cs(struct device *dev, + struct spi_device *spi, u8 idx, + struct spi_device *new_spi, u8 new_idx) +{ + u8 cs, cs_new; + u8 idx_new; + + cs = spi_get_chipselect(spi, idx); + for (idx_new = new_idx; idx_new < SPI_CS_CNT_MAX; idx_new++) { + cs_new = spi_get_chipselect(new_spi, idx_new); + if (is_valid_cs(cs) && is_valid_cs(cs_new) && cs == cs_new) { + dev_err(dev, "chipselect %u already in use\n", cs_new); + return -EBUSY; + } + } + return 0; +} + static int spi_dev_check(struct device *dev, void *data) { struct spi_device *spi = to_spi_device(dev); struct spi_device *new_spi = data; - int idx, nw_idx; - u8 cs, cs_nw; + int status, idx; if (spi->controller == new_spi->controller) { for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) { - cs = spi_get_chipselect(spi, idx); - for (nw_idx = 0; nw_idx < SPI_CS_CNT_MAX; nw_idx++) { - cs_nw = spi_get_chipselect(new_spi, nw_idx); - if (cs != 0xFF && cs_nw != 0xFF && cs == cs_nw) { - dev_err(dev, "chipselect %d already in use\n", cs_nw); - return -EBUSY; - } - } + status = spi_dev_check_cs(dev, spi, idx, new_spi, 0); + if (status) + return status; } } return 0; @@ -640,13 +668,13 @@ static int __spi_add_device(struct spi_device *spi) { struct spi_controller *ctlr = spi->controller; struct device *dev = ctlr->dev.parent; - int status, idx, nw_idx; - u8 cs, nw_cs; + int status, idx; + u8 cs; for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) { /* Chipselects are numbered 0..max; validate. */ cs = spi_get_chipselect(spi, idx); - if (cs != 0xFF && cs >= ctlr->num_chipselect) { + if (is_valid_cs(cs) && cs >= ctlr->num_chipselect) { dev_err(dev, "cs%d >= max %d\n", spi_get_chipselect(spi, idx), ctlr->num_chipselect); return -EINVAL; @@ -658,14 +686,9 @@ static int __spi_add_device(struct spi_device *spi) * For example, spi->chip_select[0] != spi->chip_select[1] and so on. */ for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) { - cs = spi_get_chipselect(spi, idx); - for (nw_idx = idx + 1; nw_idx < SPI_CS_CNT_MAX; nw_idx++) { - nw_cs = spi_get_chipselect(spi, nw_idx); - if (cs != 0xFF && nw_cs != 0xFF && cs == nw_cs) { - dev_err(dev, "chipselect %d already in use\n", nw_cs); - return -EBUSY; - } - } + status = spi_dev_check_cs(dev, spi, idx, spi, idx + 1); + if (status) + return status; } /* Set the bus ID string */ @@ -691,7 +714,7 @@ static int __spi_add_device(struct spi_device *spi) for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) { cs = spi_get_chipselect(spi, idx); - if (cs != 0xFF) + if (is_valid_cs(cs)) spi_set_csgpiod(spi, idx, ctlr->cs_gpiods[cs]); } } @@ -745,6 +768,14 @@ int spi_add_device(struct spi_device *spi) } EXPORT_SYMBOL_GPL(spi_add_device); +static void spi_set_all_cs_unused(struct spi_device *spi) +{ + u8 idx; + + for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) + spi_set_chipselect(spi, idx, SPI_INVALID_CS); +} + /** * spi_new_device - instantiate one new SPI device * @ctlr: Controller to which device is connected @@ -764,7 +795,6 @@ struct spi_device *spi_new_device(struct spi_controller *ctlr, { struct spi_device *proxy; int status; - u8 idx; /* * NOTE: caller did any chip->bus_num checks necessary. @@ -780,19 +810,10 @@ struct spi_device *spi_new_device(struct spi_controller *ctlr, WARN_ON(strlen(chip->modalias) >= sizeof(proxy->modalias)); - /* - * Zero(0) is a valid physical CS value and can be located at any - * logical CS in the spi->chip_select[]. If all the physical CS - * are initialized to 0 then It would be difficult to differentiate - * between a valid physical CS 0 & an unused logical CS whose physical - * CS can be 0. As a solution to this issue initialize all the CS to 0xFF. - * Now all the unused logical CS will have 0xFF physical CS value & can be - * ignore while performing physical CS validity checks. - */ - for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) - spi_set_chipselect(proxy, idx, 0xFF); - + /* Use provided chip-select for proxy device */ + spi_set_all_cs_unused(proxy); spi_set_chipselect(proxy, 0, chip->chip_select); + proxy->max_speed_hz = chip->max_speed_hz; proxy->mode = chip->mode; proxy->irq = chip->irq; @@ -1007,7 +1028,7 @@ static inline bool spi_is_last_cs(struct spi_device *spi) bool last = false; for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) { - if ((spi->cs_index_mask >> idx) & 0x01) { + if (spi->cs_index_mask & BIT(idx)) { if (spi->controller->last_cs[idx] == spi_get_chipselect(spi, idx)) last = true; } @@ -1036,7 +1057,7 @@ static void spi_set_cs(struct spi_device *spi, bool enable, bool force) spi->controller->last_cs_index_mask = spi->cs_index_mask; for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) - spi->controller->last_cs[idx] = enable ? spi_get_chipselect(spi, 0) : -1; + spi->controller->last_cs[idx] = enable ? spi_get_chipselect(spi, 0) : SPI_INVALID_CS; spi->controller->last_cs_mode_high = spi->mode & SPI_CS_HIGH; if (spi->mode & SPI_CS_HIGH) @@ -1058,8 +1079,7 @@ static void spi_set_cs(struct spi_device *spi, bool enable, bool force) * into account. */ for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) { - if (((spi->cs_index_mask >> idx) & 0x01) && - spi_get_csgpiod(spi, idx)) { + if ((spi->cs_index_mask & BIT(idx)) && spi_get_csgpiod(spi, idx)) { if (has_acpi_companion(&spi->dev)) gpiod_set_value_cansleep(spi_get_csgpiod(spi, idx), !enable); @@ -1747,15 +1767,6 @@ static int __spi_pump_transfer_message(struct spi_controller *ctlr, trace_spi_message_start(msg); - ret = spi_split_transfers_maxsize(ctlr, msg, - spi_max_transfer_size(msg->spi), - GFP_KERNEL | GFP_DMA); - if (ret) { - msg->status = ret; - spi_finalize_current_message(ctlr); - return ret; - } - if (ctlr->prepare_message) { ret = ctlr->prepare_message(ctlr, msg); if (ret) { @@ -2083,6 +2094,43 @@ struct spi_message *spi_get_next_queued_message(struct spi_controller *ctlr) } EXPORT_SYMBOL_GPL(spi_get_next_queued_message); +/* + * __spi_unoptimize_message - shared implementation of spi_unoptimize_message() + * and spi_maybe_unoptimize_message() + * @msg: the message to unoptimize + * + * Peripheral drivers should use spi_unoptimize_message() and callers inside + * core should use spi_maybe_unoptimize_message() rather than calling this + * function directly. + * + * It is not valid to call this on a message that is not currently optimized. + */ +static void __spi_unoptimize_message(struct spi_message *msg) +{ + struct spi_controller *ctlr = msg->spi->controller; + + if (ctlr->unoptimize_message) + ctlr->unoptimize_message(msg); + + spi_res_release(ctlr, msg); + + msg->optimized = false; + msg->opt_state = NULL; +} + +/* + * spi_maybe_unoptimize_message - unoptimize msg not managed by a peripheral + * @msg: the message to unoptimize + * + * This function is used to unoptimize a message if and only if it was + * optimized by the core (via spi_maybe_optimize_message()). + */ +static void spi_maybe_unoptimize_message(struct spi_message *msg) +{ + if (!msg->pre_optimized && msg->optimized) + __spi_unoptimize_message(msg); +} + /** * spi_finalize_current_message() - the current message is complete * @ctlr: the controller to return the message to @@ -2111,15 +2159,6 @@ void spi_finalize_current_message(struct spi_controller *ctlr) spi_unmap_msg(ctlr, mesg); - /* - * In the prepare_messages callback the SPI bus has the opportunity - * to split a transfer to smaller chunks. - * - * Release the split transfers here since spi_map_msg() is done on - * the split transfers. - */ - spi_res_release(ctlr, mesg); - if (mesg->prepared && ctlr->unprepare_message) { ret = ctlr->unprepare_message(ctlr, mesg); if (ret) { @@ -2130,6 +2169,8 @@ void spi_finalize_current_message(struct spi_controller *ctlr) mesg->prepared = false; + spi_maybe_unoptimize_message(mesg); + WRITE_ONCE(ctlr->cur_msg_incomplete, false); smp_mb(); /* See __spi_pump_transfer_message()... */ if (READ_ONCE(ctlr->cur_msg_need_completion)) @@ -2397,17 +2438,7 @@ static int of_spi_parse_dt(struct spi_controller *ctlr, struct spi_device *spi, return -EINVAL; } - /* - * Zero(0) is a valid physical CS value and can be located at any - * logical CS in the spi->chip_select[]. If all the physical CS - * are initialized to 0 then It would be difficult to differentiate - * between a valid physical CS 0 & an unused logical CS whose physical - * CS can be 0. As a solution to this issue initialize all the CS to 0xFF. - * Now all the unused logical CS will have 0xFF physical CS value & can be - * ignore while performing physical CS validity checks. - */ - for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) - spi_set_chipselect(spi, idx, 0xFF); + spi_set_all_cs_unused(spi); /* Device address */ rc = of_property_read_variable_u32_array(nc, "reg", &cs[0], 1, @@ -2431,14 +2462,10 @@ static int of_spi_parse_dt(struct spi_controller *ctlr, struct spi_device *spi, spi_set_chipselect(spi, idx, cs[idx]); /* - * spi->chip_select[i] gives the corresponding physical CS for logical CS i - * logical CS number is represented by setting the ith bit in spi->cs_index_mask - * So, for example, if spi->cs_index_mask = 0x01 then logical CS number is 0 and - * spi->chip_select[0] will give the physical CS. - * By default spi->chip_select[0] will hold the physical CS number so, set - * spi->cs_index_mask as 0x01. + * By default spi->chip_select[0] will hold the physical CS number, + * so set bit 0 in spi->cs_index_mask. */ - spi->cs_index_mask = 0x01; + spi->cs_index_mask = BIT(0); /* Device speed */ if (!of_property_read_u32(nc, "spi-max-frequency", &value)) @@ -2544,7 +2571,6 @@ struct spi_device *spi_new_ancillary_device(struct spi_device *spi, struct spi_controller *ctlr = spi->controller; struct spi_device *ancillary; int rc = 0; - u8 idx; /* Alloc an spi_device */ ancillary = spi_alloc_device(ctlr); @@ -2555,33 +2581,18 @@ struct spi_device *spi_new_ancillary_device(struct spi_device *spi, strscpy(ancillary->modalias, "dummy", sizeof(ancillary->modalias)); - /* - * Zero(0) is a valid physical CS value and can be located at any - * logical CS in the spi->chip_select[]. If all the physical CS - * are initialized to 0 then It would be difficult to differentiate - * between a valid physical CS 0 & an unused logical CS whose physical - * CS can be 0. As a solution to this issue initialize all the CS to 0xFF. - * Now all the unused logical CS will have 0xFF physical CS value & can be - * ignore while performing physical CS validity checks. - */ - for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) - spi_set_chipselect(ancillary, idx, 0xFF); - /* Use provided chip-select for ancillary device */ + spi_set_all_cs_unused(ancillary); spi_set_chipselect(ancillary, 0, chip_select); /* Take over SPI mode/speed from SPI main device */ ancillary->max_speed_hz = spi->max_speed_hz; ancillary->mode = spi->mode; /* - * spi->chip_select[i] gives the corresponding physical CS for logical CS i - * logical CS number is represented by setting the ith bit in spi->cs_index_mask - * So, for example, if spi->cs_index_mask = 0x01 then logical CS number is 0 and - * spi->chip_select[0] will give the physical CS. - * By default spi->chip_select[0] will hold the physical CS number so, set - * spi->cs_index_mask as 0x01. + * By default spi->chip_select[0] will hold the physical CS number, + * so set bit 0 in spi->cs_index_mask. */ - ancillary->cs_index_mask = 0x01; + ancillary->cs_index_mask = BIT(0); WARN_ON(!mutex_is_locked(&ctlr->add_lock)); @@ -2784,7 +2795,6 @@ struct spi_device *acpi_spi_device_alloc(struct spi_controller *ctlr, struct acpi_spi_lookup lookup = {}; struct spi_device *spi; int ret; - u8 idx; if (!ctlr && index == -1) return ERR_PTR(-EINVAL); @@ -2820,33 +2830,19 @@ struct spi_device *acpi_spi_device_alloc(struct spi_controller *ctlr, return ERR_PTR(-ENOMEM); } - /* - * Zero(0) is a valid physical CS value and can be located at any - * logical CS in the spi->chip_select[]. If all the physical CS - * are initialized to 0 then It would be difficult to differentiate - * between a valid physical CS 0 & an unused logical CS whose physical - * CS can be 0. As a solution to this issue initialize all the CS to 0xFF. - * Now all the unused logical CS will have 0xFF physical CS value & can be - * ignore while performing physical CS validity checks. - */ - for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) - spi_set_chipselect(spi, idx, 0xFF); + spi_set_all_cs_unused(spi); + spi_set_chipselect(spi, 0, lookup.chip_select); ACPI_COMPANION_SET(&spi->dev, adev); spi->max_speed_hz = lookup.max_speed_hz; spi->mode |= lookup.mode; spi->irq = lookup.irq; spi->bits_per_word = lookup.bits_per_word; - spi_set_chipselect(spi, 0, lookup.chip_select); /* - * spi->chip_select[i] gives the corresponding physical CS for logical CS i - * logical CS number is represented by setting the ith bit in spi->cs_index_mask - * So, for example, if spi->cs_index_mask = 0x01 then logical CS number is 0 and - * spi->chip_select[0] will give the physical CS. - * By default spi->chip_select[0] will hold the physical CS number so, set - * spi->cs_index_mask as 0x01. + * By default spi->chip_select[0] will hold the physical CS number, + * so set bit 0 in spi->cs_index_mask. */ - spi->cs_index_mask = 0x01; + spi->cs_index_mask = BIT(0); return spi; } @@ -3344,9 +3340,9 @@ int spi_register_controller(struct spi_controller *ctlr) goto free_bus_id; } - /* Setting last_cs to -1 means no chip selected */ + /* Setting last_cs to SPI_INVALID_CS means no chip selected */ for (idx = 0; idx < SPI_CS_CNT_MAX; idx++) - ctlr->last_cs[idx] = -1; + ctlr->last_cs[idx] = SPI_INVALID_CS; status = device_add(&ctlr->dev); if (status < 0) @@ -3687,8 +3683,7 @@ static struct spi_replaced_transfers *spi_replace_transfers( static int __spi_split_transfer_maxsize(struct spi_controller *ctlr, struct spi_message *msg, struct spi_transfer **xferp, - size_t maxsize, - gfp_t gfp) + size_t maxsize) { struct spi_transfer *xfer = *xferp, *xfers; struct spi_replaced_transfers *srt; @@ -3699,7 +3694,7 @@ static int __spi_split_transfer_maxsize(struct spi_controller *ctlr, count = DIV_ROUND_UP(xfer->len, maxsize); /* Create replacement */ - srt = spi_replace_transfers(msg, xfer, 1, count, NULL, 0, gfp); + srt = spi_replace_transfers(msg, xfer, 1, count, NULL, 0, GFP_KERNEL); if (IS_ERR(srt)) return PTR_ERR(srt); xfers = srt->inserted_transfers; @@ -3759,14 +3754,16 @@ static int __spi_split_transfer_maxsize(struct spi_controller *ctlr, * @ctlr: the @spi_controller for this transfer * @msg: the @spi_message to transform * @maxsize: the maximum when to apply this - * @gfp: GFP allocation flags + * + * This function allocates resources that are automatically freed during the + * spi message unoptimize phase so this function should only be called from + * optimize_message callbacks. * * Return: status of transformation */ int spi_split_transfers_maxsize(struct spi_controller *ctlr, struct spi_message *msg, - size_t maxsize, - gfp_t gfp) + size_t maxsize) { struct spi_transfer *xfer; int ret; @@ -3781,7 +3778,7 @@ int spi_split_transfers_maxsize(struct spi_controller *ctlr, list_for_each_entry(xfer, &msg->transfers, transfer_list) { if (xfer->len > maxsize) { ret = __spi_split_transfer_maxsize(ctlr, msg, &xfer, - maxsize, gfp); + maxsize); if (ret) return ret; } @@ -3799,14 +3796,16 @@ EXPORT_SYMBOL_GPL(spi_split_transfers_maxsize); * @ctlr: the @spi_controller for this transfer * @msg: the @spi_message to transform * @maxwords: the number of words to limit each transfer to - * @gfp: GFP allocation flags + * + * This function allocates resources that are automatically freed during the + * spi message unoptimize phase so this function should only be called from + * optimize_message callbacks. * * Return: status of transformation */ int spi_split_transfers_maxwords(struct spi_controller *ctlr, struct spi_message *msg, - size_t maxwords, - gfp_t gfp) + size_t maxwords) { struct spi_transfer *xfer; @@ -3824,7 +3823,7 @@ int spi_split_transfers_maxwords(struct spi_controller *ctlr, maxsize = maxwords * roundup_pow_of_two(BITS_TO_BYTES(xfer->bits_per_word)); if (xfer->len > maxsize) { ret = __spi_split_transfer_maxsize(ctlr, msg, &xfer, - maxsize, gfp); + maxsize); if (ret) return ret; } @@ -4063,33 +4062,7 @@ static int __spi_validate(struct spi_device *spi, struct spi_message *message) if (list_empty(&message->transfers)) return -EINVAL; - /* - * If an SPI controller does not support toggling the CS line on each - * transfer (indicated by the SPI_CS_WORD flag) or we are using a GPIO - * for the CS line, we can emulate the CS-per-word hardware function by - * splitting transfers into one-word transfers and ensuring that - * cs_change is set for each transfer. - */ - if ((spi->mode & SPI_CS_WORD) && (!(ctlr->mode_bits & SPI_CS_WORD) || - spi_is_csgpiod(spi))) { - size_t maxsize = BITS_TO_BYTES(spi->bits_per_word); - int ret; - - /* spi_split_transfers_maxsize() requires message->spi */ - message->spi = spi; - - ret = spi_split_transfers_maxsize(ctlr, message, maxsize, - GFP_KERNEL); - if (ret) - return ret; - - list_for_each_entry(xfer, &message->transfers, transfer_list) { - /* Don't change cs_change on the last entry in the list */ - if (list_is_last(&xfer->transfer_list, &message->transfers)) - break; - xfer->cs_change = 1; - } - } + message->spi = spi; /* * Half-duplex links include original MicroWire, and ones with @@ -4202,6 +4175,167 @@ static int __spi_validate(struct spi_device *spi, struct spi_message *message) return 0; } +/* + * spi_split_transfers - generic handling of transfer splitting + * @msg: the message to split + * + * Under certain conditions, a SPI controller may not support arbitrary + * transfer sizes or other features required by a peripheral. This function + * will split the transfers in the message into smaller transfers that are + * supported by the controller. + * + * Controllers with special requirements not covered here can also split + * transfers in the optimize_message() callback. + * + * Context: can sleep + * Return: zero on success, else a negative error code + */ +static int spi_split_transfers(struct spi_message *msg) +{ + struct spi_controller *ctlr = msg->spi->controller; + struct spi_transfer *xfer; + int ret; + + /* + * If an SPI controller does not support toggling the CS line on each + * transfer (indicated by the SPI_CS_WORD flag) or we are using a GPIO + * for the CS line, we can emulate the CS-per-word hardware function by + * splitting transfers into one-word transfers and ensuring that + * cs_change is set for each transfer. + */ + if ((msg->spi->mode & SPI_CS_WORD) && + (!(ctlr->mode_bits & SPI_CS_WORD) || spi_is_csgpiod(msg->spi))) { + ret = spi_split_transfers_maxwords(ctlr, msg, 1); + if (ret) + return ret; + + list_for_each_entry(xfer, &msg->transfers, transfer_list) { + /* Don't change cs_change on the last entry in the list */ + if (list_is_last(&xfer->transfer_list, &msg->transfers)) + break; + + xfer->cs_change = 1; + } + } else { + ret = spi_split_transfers_maxsize(ctlr, msg, + spi_max_transfer_size(msg->spi)); + if (ret) + return ret; + } + + return 0; +} + +/* + * __spi_optimize_message - shared implementation for spi_optimize_message() + * and spi_maybe_optimize_message() + * @spi: the device that will be used for the message + * @msg: the message to optimize + * + * Peripheral drivers will call spi_optimize_message() and the spi core will + * call spi_maybe_optimize_message() instead of calling this directly. + * + * It is not valid to call this on a message that has already been optimized. + * + * Return: zero on success, else a negative error code + */ +static int __spi_optimize_message(struct spi_device *spi, + struct spi_message *msg) +{ + struct spi_controller *ctlr = spi->controller; + int ret; + + ret = __spi_validate(spi, msg); + if (ret) + return ret; + + ret = spi_split_transfers(msg); + if (ret) + return ret; + + if (ctlr->optimize_message) { + ret = ctlr->optimize_message(msg); + if (ret) { + spi_res_release(ctlr, msg); + return ret; + } + } + + msg->optimized = true; + + return 0; +} + +/* + * spi_maybe_optimize_message - optimize message if it isn't already pre-optimized + * @spi: the device that will be used for the message + * @msg: the message to optimize + * Return: zero on success, else a negative error code + */ +static int spi_maybe_optimize_message(struct spi_device *spi, + struct spi_message *msg) +{ + if (msg->pre_optimized) + return 0; + + return __spi_optimize_message(spi, msg); +} + +/** + * spi_optimize_message - do any one-time validation and setup for a SPI message + * @spi: the device that will be used for the message + * @msg: the message to optimize + * + * Peripheral drivers that reuse the same message repeatedly may call this to + * perform as much message prep as possible once, rather than repeating it each + * time a message transfer is performed to improve throughput and reduce CPU + * usage. + * + * Once a message has been optimized, it cannot be modified with the exception + * of updating the contents of any xfer->tx_buf (the pointer can't be changed, + * only the data in the memory it points to). + * + * Calls to this function must be balanced with calls to spi_unoptimize_message() + * to avoid leaking resources. + * + * Context: can sleep + * Return: zero on success, else a negative error code + */ +int spi_optimize_message(struct spi_device *spi, struct spi_message *msg) +{ + int ret; + + ret = __spi_optimize_message(spi, msg); + if (ret) + return ret; + + /* + * This flag indicates that the peripheral driver called spi_optimize_message() + * and therefore we shouldn't unoptimize message automatically when finalizing + * the message but rather wait until spi_unoptimize_message() is called + * by the peripheral driver. + */ + msg->pre_optimized = true; + + return 0; +} +EXPORT_SYMBOL_GPL(spi_optimize_message); + +/** + * spi_unoptimize_message - releases any resources allocated by spi_optimize_message() + * @msg: the message to unoptimize + * + * Calls to this function must be balanced with calls to spi_optimize_message(). + * + * Context: can sleep + */ +void spi_unoptimize_message(struct spi_message *msg) +{ + __spi_unoptimize_message(msg); + msg->pre_optimized = false; +} +EXPORT_SYMBOL_GPL(spi_unoptimize_message); + static int __spi_async(struct spi_device *spi, struct spi_message *message) { struct spi_controller *ctlr = spi->controller; @@ -4214,8 +4348,6 @@ static int __spi_async(struct spi_device *spi, struct spi_message *message) if (!ctlr->transfer) return -ENOTSUPP; - message->spi = spi; - SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, spi_async); SPI_STATISTICS_INCREMENT_FIELD(spi->pcpu_statistics, spi_async); @@ -4268,8 +4400,8 @@ int spi_async(struct spi_device *spi, struct spi_message *message) int ret; unsigned long flags; - ret = __spi_validate(spi, message); - if (ret != 0) + ret = spi_maybe_optimize_message(spi, message); + if (ret) return ret; spin_lock_irqsave(&ctlr->bus_lock_spinlock, flags); @@ -4281,61 +4413,12 @@ int spi_async(struct spi_device *spi, struct spi_message *message) spin_unlock_irqrestore(&ctlr->bus_lock_spinlock, flags); + spi_maybe_unoptimize_message(message); + return ret; } EXPORT_SYMBOL_GPL(spi_async); -/** - * spi_async_locked - version of spi_async with exclusive bus usage - * @spi: device with which data will be exchanged - * @message: describes the data transfers, including completion callback - * Context: any (IRQs may be blocked, etc) - * - * This call may be used in_irq and other contexts which can't sleep, - * as well as from task contexts which can sleep. - * - * The completion callback is invoked in a context which can't sleep. - * Before that invocation, the value of message->status is undefined. - * When the callback is issued, message->status holds either zero (to - * indicate complete success) or a negative error code. After that - * callback returns, the driver which issued the transfer request may - * deallocate the associated memory; it's no longer in use by any SPI - * core or controller driver code. - * - * Note that although all messages to a spi_device are handled in - * FIFO order, messages may go to different devices in other orders. - * Some device might be higher priority, or have various "hard" access - * time requirements, for example. - * - * On detection of any fault during the transfer, processing of - * the entire message is aborted, and the device is deselected. - * Until returning from the associated message completion callback, - * no other spi_message queued to that device will be processed. - * (This rule applies equally to all the synchronous transfer calls, - * which are wrappers around this core asynchronous primitive.) - * - * Return: zero on success, else a negative error code. - */ -static int spi_async_locked(struct spi_device *spi, struct spi_message *message) -{ - struct spi_controller *ctlr = spi->controller; - int ret; - unsigned long flags; - - ret = __spi_validate(spi, message); - if (ret != 0) - return ret; - - spin_lock_irqsave(&ctlr->bus_lock_spinlock, flags); - - ret = __spi_async(spi, message); - - spin_unlock_irqrestore(&ctlr->bus_lock_spinlock, flags); - - return ret; - -} - static void __spi_transfer_message_noqueue(struct spi_controller *ctlr, struct spi_message *msg) { bool was_busy; @@ -4383,6 +4466,7 @@ static void spi_complete(void *arg) static int __spi_sync(struct spi_device *spi, struct spi_message *message) { DECLARE_COMPLETION_ONSTACK(done); + unsigned long flags; int status; struct spi_controller *ctlr = spi->controller; @@ -4391,12 +4475,10 @@ static int __spi_sync(struct spi_device *spi, struct spi_message *message) return -ESHUTDOWN; } - status = __spi_validate(spi, message); - if (status != 0) + status = spi_maybe_optimize_message(spi, message); + if (status) return status; - message->spi = spi; - SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, spi_sync); SPI_STATISTICS_INCREMENT_FIELD(spi->pcpu_statistics, spi_sync); @@ -4428,7 +4510,11 @@ static int __spi_sync(struct spi_device *spi, struct spi_message *message) */ message->complete = spi_complete; message->context = &done; - status = spi_async_locked(spi, message); + + spin_lock_irqsave(&ctlr->bus_lock_spinlock, flags); + status = __spi_async(spi, message); + spin_unlock_irqrestore(&ctlr->bus_lock_spinlock, flags); + if (status == 0) { wait_for_completion(&done); status = message->status; diff --git a/drivers/spi/spidev.c b/drivers/spi/spidev.c index 655f2c959cd4..95fb5f1c91c1 100644 --- a/drivers/spi/spidev.c +++ b/drivers/spi/spidev.c @@ -802,7 +802,7 @@ static int spidev_probe(struct spi_device *spi) spidev->devt = MKDEV(SPIDEV_MAJOR, minor); dev = device_create(&spidev_class, &spi->dev, spidev->devt, spidev, "spidev%d.%d", - spi->master->bus_num, spi_get_chipselect(spi, 0)); + spi->controller->bus_num, spi_get_chipselect(spi, 0)); status = PTR_ERR_OR_ZERO(dev); } else { dev_dbg(&spi->dev, "no minor number available!\n"); diff --git a/drivers/staging/fbtft/fbtft-core.c b/drivers/staging/fbtft/fbtft-core.c index 3626f429b002..68add4d598ae 100644 --- a/drivers/staging/fbtft/fbtft-core.c +++ b/drivers/staging/fbtft/fbtft-core.c @@ -794,7 +794,7 @@ int fbtft_register_framebuffer(struct fb_info *fb_info) if (par->txbuf.buf && par->txbuf.len >= 1024) sprintf(text1, ", %zu KiB buffer memory", par->txbuf.len >> 10); if (spi) - sprintf(text2, ", spi%d.%d at %d MHz", spi->master->bus_num, + sprintf(text2, ", spi%d.%d at %d MHz", spi->controller->bus_num, spi_get_chipselect(spi, 0), spi->max_speed_hz / 1000000); dev_info(fb_info->dev, "%s frame buffer, %dx%d, %d KiB video memory%s, fps=%lu%s\n", @@ -1215,7 +1215,7 @@ int fbtft_probe_common(struct fbtft_display *display, /* 9-bit SPI setup */ if (par->spi && display->buswidth == 9) { - if (par->spi->master->bits_per_word_mask & SPI_BPW_MASK(9)) { + if (par->spi->controller->bits_per_word_mask & SPI_BPW_MASK(9)) { par->spi->bits_per_word = 9; } else { dev_warn(&par->spi->dev, diff --git a/drivers/staging/greybus/pwm.c b/drivers/staging/greybus/pwm.c index a3cb68cfa0f9..01883fbcd79b 100644 --- a/drivers/staging/greybus/pwm.c +++ b/drivers/staging/greybus/pwm.c @@ -16,8 +16,6 @@ struct gb_pwm_chip { struct gb_connection *connection; - u8 pwm_max; /* max pwm number */ - struct pwm_chip chip; }; @@ -26,32 +24,33 @@ static inline struct gb_pwm_chip *pwm_chip_to_gb_pwm_chip(struct pwm_chip *chip) return container_of(chip, struct gb_pwm_chip, chip); } -static int gb_pwm_count_operation(struct gb_pwm_chip *pwmc) +static int gb_pwm_get_npwm(struct gb_connection *connection) { struct gb_pwm_count_response response; int ret; - ret = gb_operation_sync(pwmc->connection, GB_PWM_TYPE_PWM_COUNT, + ret = gb_operation_sync(connection, GB_PWM_TYPE_PWM_COUNT, NULL, 0, &response, sizeof(response)); if (ret) return ret; - pwmc->pwm_max = response.count; - return 0; + + /* + * The request returns the highest allowed PWM id parameter. So add one + * to get the number of PWMs. + */ + return response.count + 1; } -static int gb_pwm_activate_operation(struct gb_pwm_chip *pwmc, - u8 which) +static int gb_pwm_activate_operation(struct pwm_chip *chip, u8 which) { + struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); struct gb_pwm_activate_request request; struct gbphy_device *gbphy_dev; int ret; - if (which > pwmc->pwm_max) - return -EINVAL; - request.which = which; - gbphy_dev = to_gbphy_dev(pwmc->chip.dev); + gbphy_dev = to_gbphy_dev(pwmchip_parent(chip)); ret = gbphy_runtime_get_sync(gbphy_dev); if (ret) return ret; @@ -64,19 +63,16 @@ static int gb_pwm_activate_operation(struct gb_pwm_chip *pwmc, return ret; } -static int gb_pwm_deactivate_operation(struct gb_pwm_chip *pwmc, - u8 which) +static int gb_pwm_deactivate_operation(struct pwm_chip *chip, u8 which) { + struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); struct gb_pwm_deactivate_request request; struct gbphy_device *gbphy_dev; int ret; - if (which > pwmc->pwm_max) - return -EINVAL; - request.which = which; - gbphy_dev = to_gbphy_dev(pwmc->chip.dev); + gbphy_dev = to_gbphy_dev(pwmchip_parent(chip)); ret = gbphy_runtime_get_sync(gbphy_dev); if (ret) return ret; @@ -89,21 +85,19 @@ static int gb_pwm_deactivate_operation(struct gb_pwm_chip *pwmc, return ret; } -static int gb_pwm_config_operation(struct gb_pwm_chip *pwmc, +static int gb_pwm_config_operation(struct pwm_chip *chip, u8 which, u32 duty, u32 period) { + struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); struct gb_pwm_config_request request; struct gbphy_device *gbphy_dev; int ret; - if (which > pwmc->pwm_max) - return -EINVAL; - request.which = which; request.duty = cpu_to_le32(duty); request.period = cpu_to_le32(period); - gbphy_dev = to_gbphy_dev(pwmc->chip.dev); + gbphy_dev = to_gbphy_dev(pwmchip_parent(chip)); ret = gbphy_runtime_get_sync(gbphy_dev); if (ret) return ret; @@ -116,20 +110,18 @@ static int gb_pwm_config_operation(struct gb_pwm_chip *pwmc, return ret; } -static int gb_pwm_set_polarity_operation(struct gb_pwm_chip *pwmc, +static int gb_pwm_set_polarity_operation(struct pwm_chip *chip, u8 which, u8 polarity) { + struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); struct gb_pwm_polarity_request request; struct gbphy_device *gbphy_dev; int ret; - if (which > pwmc->pwm_max) - return -EINVAL; - request.which = which; request.polarity = polarity; - gbphy_dev = to_gbphy_dev(pwmc->chip.dev); + gbphy_dev = to_gbphy_dev(pwmchip_parent(chip)); ret = gbphy_runtime_get_sync(gbphy_dev); if (ret) return ret; @@ -142,19 +134,16 @@ static int gb_pwm_set_polarity_operation(struct gb_pwm_chip *pwmc, return ret; } -static int gb_pwm_enable_operation(struct gb_pwm_chip *pwmc, - u8 which) +static int gb_pwm_enable_operation(struct pwm_chip *chip, u8 which) { + struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); struct gb_pwm_enable_request request; struct gbphy_device *gbphy_dev; int ret; - if (which > pwmc->pwm_max) - return -EINVAL; - request.which = which; - gbphy_dev = to_gbphy_dev(pwmc->chip.dev); + gbphy_dev = to_gbphy_dev(pwmchip_parent(chip)); ret = gbphy_runtime_get_sync(gbphy_dev); if (ret) return ret; @@ -167,22 +156,19 @@ static int gb_pwm_enable_operation(struct gb_pwm_chip *pwmc, return ret; } -static int gb_pwm_disable_operation(struct gb_pwm_chip *pwmc, - u8 which) +static int gb_pwm_disable_operation(struct pwm_chip *chip, u8 which) { + struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); struct gb_pwm_disable_request request; struct gbphy_device *gbphy_dev; int ret; - if (which > pwmc->pwm_max) - return -EINVAL; - request.which = which; ret = gb_operation_sync(pwmc->connection, GB_PWM_TYPE_DISABLE, &request, sizeof(request), NULL, 0); - gbphy_dev = to_gbphy_dev(pwmc->chip.dev); + gbphy_dev = to_gbphy_dev(pwmchip_parent(chip)); gbphy_runtime_put_autosuspend(gbphy_dev); return ret; @@ -190,19 +176,15 @@ static int gb_pwm_disable_operation(struct gb_pwm_chip *pwmc, static int gb_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm) { - struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); - - return gb_pwm_activate_operation(pwmc, pwm->hwpwm); + return gb_pwm_activate_operation(chip, pwm->hwpwm); }; static void gb_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm) { - struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); - if (pwm_is_enabled(pwm)) - dev_warn(chip->dev, "freeing PWM device without disabling\n"); + dev_warn(pwmchip_parent(chip), "freeing PWM device without disabling\n"); - gb_pwm_deactivate_operation(pwmc, pwm->hwpwm); + gb_pwm_deactivate_operation(chip, pwm->hwpwm); } static int gb_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, @@ -212,22 +194,21 @@ static int gb_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, bool enabled = pwm->state.enabled; u64 period = state->period; u64 duty_cycle = state->duty_cycle; - struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); /* Set polarity */ if (state->polarity != pwm->state.polarity) { if (enabled) { - gb_pwm_disable_operation(pwmc, pwm->hwpwm); + gb_pwm_disable_operation(chip, pwm->hwpwm); enabled = false; } - err = gb_pwm_set_polarity_operation(pwmc, pwm->hwpwm, state->polarity); + err = gb_pwm_set_polarity_operation(chip, pwm->hwpwm, state->polarity); if (err) return err; } if (!state->enabled) { if (enabled) - gb_pwm_disable_operation(pwmc, pwm->hwpwm); + gb_pwm_disable_operation(chip, pwm->hwpwm); return 0; } @@ -243,13 +224,13 @@ static int gb_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, if (duty_cycle > period) duty_cycle = period; - err = gb_pwm_config_operation(pwmc, pwm->hwpwm, duty_cycle, period); + err = gb_pwm_config_operation(chip, pwm->hwpwm, duty_cycle, period); if (err) return err; /* enable/disable */ if (!enabled) - return gb_pwm_enable_operation(pwmc, pwm->hwpwm); + return gb_pwm_enable_operation(chip, pwm->hwpwm); return 0; } @@ -266,61 +247,59 @@ static int gb_pwm_probe(struct gbphy_device *gbphy_dev, struct gb_connection *connection; struct gb_pwm_chip *pwmc; struct pwm_chip *chip; - int ret; - - pwmc = kzalloc(sizeof(*pwmc), GFP_KERNEL); - if (!pwmc) - return -ENOMEM; + int ret, npwm; connection = gb_connection_create(gbphy_dev->bundle, le16_to_cpu(gbphy_dev->cport_desc->id), NULL); - if (IS_ERR(connection)) { - ret = PTR_ERR(connection); - goto exit_pwmc_free; - } - - pwmc->connection = connection; - gb_connection_set_data(connection, pwmc); - gb_gbphy_set_data(gbphy_dev, pwmc); + if (IS_ERR(connection)) + return PTR_ERR(connection); ret = gb_connection_enable(connection); if (ret) goto exit_connection_destroy; /* Query number of pwms present */ - ret = gb_pwm_count_operation(pwmc); - if (ret) + ret = gb_pwm_get_npwm(connection); + if (ret < 0) goto exit_connection_disable; + npwm = ret; - chip = &pwmc->chip; + chip = pwmchip_alloc(&gbphy_dev->dev, npwm, sizeof(*pwmc)); + if (IS_ERR(chip)) { + ret = PTR_ERR(chip); + goto exit_connection_disable; + } + gb_gbphy_set_data(gbphy_dev, chip); + + pwmc = pwm_chip_to_gb_pwm_chip(chip); + pwmc->connection = connection; - chip->dev = &gbphy_dev->dev; chip->ops = &gb_pwm_ops; - chip->npwm = pwmc->pwm_max + 1; ret = pwmchip_add(chip); if (ret) { dev_err(&gbphy_dev->dev, "failed to register PWM: %d\n", ret); - goto exit_connection_disable; + goto exit_pwmchip_put; } gbphy_runtime_put_autosuspend(gbphy_dev); return 0; +exit_pwmchip_put: + pwmchip_put(chip); exit_connection_disable: gb_connection_disable(connection); exit_connection_destroy: gb_connection_destroy(connection); -exit_pwmc_free: - kfree(pwmc); return ret; } static void gb_pwm_remove(struct gbphy_device *gbphy_dev) { - struct gb_pwm_chip *pwmc = gb_gbphy_get_data(gbphy_dev); + struct pwm_chip *chip = gb_gbphy_get_data(gbphy_dev); + struct gb_pwm_chip *pwmc = pwm_chip_to_gb_pwm_chip(chip); struct gb_connection *connection = pwmc->connection; int ret; @@ -328,10 +307,10 @@ static void gb_pwm_remove(struct gbphy_device *gbphy_dev) if (ret) gbphy_runtime_get_noresume(gbphy_dev); - pwmchip_remove(&pwmc->chip); + pwmchip_remove(chip); + pwmchip_put(chip); gb_connection_disable(connection); gb_connection_destroy(connection); - kfree(pwmc); } static const struct gbphy_device_id gb_pwm_id_table[] = { diff --git a/drivers/staging/greybus/spilib.c b/drivers/staging/greybus/spilib.c index efb3bec58e15..34f10685139f 100644 --- a/drivers/staging/greybus/spilib.c +++ b/drivers/staging/greybus/spilib.c @@ -42,7 +42,7 @@ struct gb_spilib { #define XFER_TIMEOUT_TOLERANCE 200 -static struct spi_master *get_master_from_spi(struct gb_spilib *spi) +static struct spi_controller *get_controller_from_spi(struct gb_spilib *spi) { return gb_connection_get_data(spi->connection); } @@ -324,10 +324,10 @@ static void gb_spi_decode_response(struct gb_spilib *spi, } } -static int gb_spi_transfer_one_message(struct spi_master *master, +static int gb_spi_transfer_one_message(struct spi_controller *ctlr, struct spi_message *msg) { - struct gb_spilib *spi = spi_master_get_devdata(master); + struct gb_spilib *spi = spi_controller_get_devdata(ctlr); struct gb_connection *connection = spi->connection; struct gb_spi_transfer_response *response; struct gb_operation *operation; @@ -371,21 +371,21 @@ static int gb_spi_transfer_one_message(struct spi_master *master, out: msg->status = ret; clean_xfer_state(spi); - spi_finalize_current_message(master); + spi_finalize_current_message(ctlr); return ret; } -static int gb_spi_prepare_transfer_hardware(struct spi_master *master) +static int gb_spi_prepare_transfer_hardware(struct spi_controller *ctlr) { - struct gb_spilib *spi = spi_master_get_devdata(master); + struct gb_spilib *spi = spi_controller_get_devdata(ctlr); return spi->ops->prepare_transfer_hardware(spi->parent); } -static int gb_spi_unprepare_transfer_hardware(struct spi_master *master) +static int gb_spi_unprepare_transfer_hardware(struct spi_controller *ctlr) { - struct gb_spilib *spi = spi_master_get_devdata(master); + struct gb_spilib *spi = spi_controller_get_devdata(ctlr); spi->ops->unprepare_transfer_hardware(spi->parent); @@ -440,7 +440,7 @@ static int gb_spi_get_master_config(struct gb_spilib *spi) static int gb_spi_setup_device(struct gb_spilib *spi, u8 cs) { - struct spi_master *master = get_master_from_spi(spi); + struct spi_controller *ctlr = get_controller_from_spi(spi); struct gb_spi_device_config_request request; struct gb_spi_device_config_response response; struct spi_board_info spi_board = { {0} }; @@ -471,11 +471,11 @@ static int gb_spi_setup_device(struct gb_spilib *spi, u8 cs) return -EINVAL; spi_board.mode = le16_to_cpu(response.mode); - spi_board.bus_num = master->bus_num; + spi_board.bus_num = ctlr->bus_num; spi_board.chip_select = cs; spi_board.max_speed_hz = le32_to_cpu(response.max_speed_hz); - spidev = spi_new_device(master, &spi_board); + spidev = spi_new_device(ctlr, &spi_board); if (!spidev) return -EINVAL; @@ -486,52 +486,52 @@ int gb_spilib_master_init(struct gb_connection *connection, struct device *dev, struct spilib_ops *ops) { struct gb_spilib *spi; - struct spi_master *master; + struct spi_controller *ctlr; int ret; u8 i; /* Allocate master with space for data */ - master = spi_alloc_master(dev, sizeof(*spi)); - if (!master) { + ctlr = spi_alloc_master(dev, sizeof(*spi)); + if (!ctlr) { dev_err(dev, "cannot alloc SPI master\n"); return -ENOMEM; } - spi = spi_master_get_devdata(master); + spi = spi_controller_get_devdata(ctlr); spi->connection = connection; - gb_connection_set_data(connection, master); + gb_connection_set_data(connection, ctlr); spi->parent = dev; spi->ops = ops; - /* get master configuration */ + /* get controller configuration */ ret = gb_spi_get_master_config(spi); if (ret) goto exit_spi_put; - master->bus_num = -1; /* Allow spi-core to allocate it dynamically */ - master->num_chipselect = spi->num_chipselect; - master->mode_bits = spi->mode; - master->flags = spi->flags; - master->bits_per_word_mask = spi->bits_per_word_mask; + ctlr->bus_num = -1; /* Allow spi-core to allocate it dynamically */ + ctlr->num_chipselect = spi->num_chipselect; + ctlr->mode_bits = spi->mode; + ctlr->flags = spi->flags; + ctlr->bits_per_word_mask = spi->bits_per_word_mask; /* Attach methods */ - master->cleanup = gb_spi_cleanup; - master->setup = gb_spi_setup; - master->transfer_one_message = gb_spi_transfer_one_message; + ctlr->cleanup = gb_spi_cleanup; + ctlr->setup = gb_spi_setup; + ctlr->transfer_one_message = gb_spi_transfer_one_message; if (ops && ops->prepare_transfer_hardware) { - master->prepare_transfer_hardware = + ctlr->prepare_transfer_hardware = gb_spi_prepare_transfer_hardware; } if (ops && ops->unprepare_transfer_hardware) { - master->unprepare_transfer_hardware = + ctlr->unprepare_transfer_hardware = gb_spi_unprepare_transfer_hardware; } - master->auto_runtime_pm = true; + ctlr->auto_runtime_pm = true; - ret = spi_register_master(master); + ret = spi_register_controller(ctlr); if (ret < 0) goto exit_spi_put; @@ -548,12 +548,12 @@ int gb_spilib_master_init(struct gb_connection *connection, struct device *dev, return 0; exit_spi_put: - spi_master_put(master); + spi_controller_put(ctlr); return ret; exit_spi_unregister: - spi_unregister_master(master); + spi_unregister_controller(ctlr); return ret; } @@ -561,9 +561,9 @@ EXPORT_SYMBOL_GPL(gb_spilib_master_init); void gb_spilib_master_exit(struct gb_connection *connection) { - struct spi_master *master = gb_connection_get_data(connection); + struct spi_controller *ctlr = gb_connection_get_data(connection); - spi_unregister_master(master); + spi_unregister_controller(ctlr); } EXPORT_SYMBOL_GPL(gb_spilib_master_exit); diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index 17a8ae5e991d..204ed89a3ec9 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -83,17 +83,6 @@ config THERMAL_OF Say 'Y' here if you need to build thermal infrastructure based on device tree. -config THERMAL_WRITABLE_TRIPS - bool "Enable writable trip points" - help - This option allows the system integrator to choose whether - trip temperatures can be changed from userspace. The - writable trips need to be specified when setting up the - thermal zone but the choice here takes precedence. - - Say 'Y' here if you would like to allow userspace tools to - change trip temperatures. - choice prompt "Default Thermal governor" default THERMAL_DEFAULT_GOV_STEP_WISE diff --git a/drivers/thermal/cpufreq_cooling.c b/drivers/thermal/cpufreq_cooling.c index e2cc7bd30862..9d1b1459700d 100644 --- a/drivers/thermal/cpufreq_cooling.c +++ b/drivers/thermal/cpufreq_cooling.c @@ -91,12 +91,16 @@ struct cpufreq_cooling_device { static unsigned long get_level(struct cpufreq_cooling_device *cpufreq_cdev, unsigned int freq) { + struct em_perf_state *table; int i; + rcu_read_lock(); + table = em_perf_state_from_pd(cpufreq_cdev->em); for (i = cpufreq_cdev->max_level - 1; i >= 0; i--) { - if (freq > cpufreq_cdev->em->table[i].frequency) + if (freq > table[i].frequency) break; } + rcu_read_unlock(); return cpufreq_cdev->max_level - i - 1; } @@ -104,16 +108,20 @@ static unsigned long get_level(struct cpufreq_cooling_device *cpufreq_cdev, static u32 cpu_freq_to_power(struct cpufreq_cooling_device *cpufreq_cdev, u32 freq) { + struct em_perf_state *table; unsigned long power_mw; int i; + rcu_read_lock(); + table = em_perf_state_from_pd(cpufreq_cdev->em); for (i = cpufreq_cdev->max_level - 1; i >= 0; i--) { - if (freq > cpufreq_cdev->em->table[i].frequency) + if (freq > table[i].frequency) break; } - power_mw = cpufreq_cdev->em->table[i + 1].power; + power_mw = table[i + 1].power; power_mw /= MICROWATT_PER_MILLIWATT; + rcu_read_unlock(); return power_mw; } @@ -121,18 +129,24 @@ static u32 cpu_freq_to_power(struct cpufreq_cooling_device *cpufreq_cdev, static u32 cpu_power_to_freq(struct cpufreq_cooling_device *cpufreq_cdev, u32 power) { + struct em_perf_state *table; unsigned long em_power_mw; + u32 freq; int i; + rcu_read_lock(); + table = em_perf_state_from_pd(cpufreq_cdev->em); for (i = cpufreq_cdev->max_level; i > 0; i--) { /* Convert EM power to milli-Watts to make safe comparison */ - em_power_mw = cpufreq_cdev->em->table[i].power; + em_power_mw = table[i].power; em_power_mw /= MICROWATT_PER_MILLIWATT; if (power >= em_power_mw) break; } + freq = table[i].frequency; + rcu_read_unlock(); - return cpufreq_cdev->em->table[i].frequency; + return freq; } /** @@ -262,8 +276,9 @@ static int cpufreq_get_requested_power(struct thermal_cooling_device *cdev, static int cpufreq_state2power(struct thermal_cooling_device *cdev, unsigned long state, u32 *power) { - unsigned int freq, num_cpus, idx; struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata; + unsigned int freq, num_cpus, idx; + struct em_perf_state *table; /* Request state should be less than max_level */ if (state > cpufreq_cdev->max_level) @@ -272,7 +287,12 @@ static int cpufreq_state2power(struct thermal_cooling_device *cdev, num_cpus = cpumask_weight(cpufreq_cdev->policy->cpus); idx = cpufreq_cdev->max_level - state; - freq = cpufreq_cdev->em->table[idx].frequency; + + rcu_read_lock(); + table = em_perf_state_from_pd(cpufreq_cdev->em); + freq = table[idx].frequency; + rcu_read_unlock(); + *power = cpu_freq_to_power(cpufreq_cdev, freq) * num_cpus; return 0; @@ -378,8 +398,17 @@ static unsigned int get_state_freq(struct cpufreq_cooling_device *cpufreq_cdev, #ifdef CONFIG_THERMAL_GOV_POWER_ALLOCATOR /* Use the Energy Model table if available */ if (cpufreq_cdev->em) { + struct em_perf_state *table; + unsigned int freq; + idx = cpufreq_cdev->max_level - state; - return cpufreq_cdev->em->table[idx].frequency; + + rcu_read_lock(); + table = em_perf_state_from_pd(cpufreq_cdev->em); + freq = table[idx].frequency; + rcu_read_unlock(); + + return freq; } #endif diff --git a/drivers/thermal/da9062-thermal.c b/drivers/thermal/da9062-thermal.c index 160d64913057..a27aff88cd96 100644 --- a/drivers/thermal/da9062-thermal.c +++ b/drivers/thermal/da9062-thermal.c @@ -197,7 +197,7 @@ static int da9062_thermal_probe(struct platform_device *pdev) mutex_init(&thermal->lock); thermal->zone = thermal_zone_device_register_with_trips(thermal->config->name, - trips, ARRAY_SIZE(trips), 0, thermal, + trips, ARRAY_SIZE(trips), thermal, &da9062_thermal_ops, NULL, pp_tmp, 0); if (IS_ERR(thermal->zone)) { diff --git a/drivers/thermal/devfreq_cooling.c b/drivers/thermal/devfreq_cooling.c index 262e62ab6cf2..50dec24e967a 100644 --- a/drivers/thermal/devfreq_cooling.c +++ b/drivers/thermal/devfreq_cooling.c @@ -87,6 +87,7 @@ static int devfreq_cooling_set_cur_state(struct thermal_cooling_device *cdev, struct devfreq_cooling_device *dfc = cdev->devdata; struct devfreq *df = dfc->devfreq; struct device *dev = df->dev.parent; + struct em_perf_state *table; unsigned long freq; int perf_idx; @@ -100,7 +101,11 @@ static int devfreq_cooling_set_cur_state(struct thermal_cooling_device *cdev, if (dfc->em_pd) { perf_idx = dfc->max_state - state; - freq = dfc->em_pd->table[perf_idx].frequency * 1000; + + rcu_read_lock(); + table = em_perf_state_from_pd(dfc->em_pd); + freq = table[perf_idx].frequency * 1000; + rcu_read_unlock(); } else { freq = dfc->freq_table[state]; } @@ -123,14 +128,21 @@ static int devfreq_cooling_set_cur_state(struct thermal_cooling_device *cdev, */ static int get_perf_idx(struct em_perf_domain *em_pd, unsigned long freq) { - int i; + struct em_perf_state *table; + int i, idx = -EINVAL; + rcu_read_lock(); + table = em_perf_state_from_pd(em_pd); for (i = 0; i < em_pd->nr_perf_states; i++) { - if (em_pd->table[i].frequency == freq) - return i; - } + if (table[i].frequency != freq) + continue; - return -EINVAL; + idx = i; + break; + } + rcu_read_unlock(); + + return idx; } static unsigned long get_voltage(struct devfreq *df, unsigned long freq) @@ -181,6 +193,7 @@ static int devfreq_cooling_get_requested_power(struct thermal_cooling_device *cd struct devfreq_cooling_device *dfc = cdev->devdata; struct devfreq *df = dfc->devfreq; struct devfreq_dev_status status; + struct em_perf_state *table; unsigned long state; unsigned long freq; unsigned long voltage; @@ -204,7 +217,11 @@ static int devfreq_cooling_get_requested_power(struct thermal_cooling_device *cd state = dfc->capped_state; /* Convert EM power into milli-Watts first */ - dfc->res_util = dfc->em_pd->table[state].power; + rcu_read_lock(); + table = em_perf_state_from_pd(dfc->em_pd); + dfc->res_util = table[state].power; + rcu_read_unlock(); + dfc->res_util /= MICROWATT_PER_MILLIWATT; dfc->res_util *= SCALE_ERROR_MITIGATION; @@ -225,7 +242,11 @@ static int devfreq_cooling_get_requested_power(struct thermal_cooling_device *cd _normalize_load(&status); /* Convert EM power into milli-Watts first */ - *power = dfc->em_pd->table[perf_idx].power; + rcu_read_lock(); + table = em_perf_state_from_pd(dfc->em_pd); + *power = table[perf_idx].power; + rcu_read_unlock(); + *power /= MICROWATT_PER_MILLIWATT; /* Scale power for utilization */ *power *= status.busy_time; @@ -245,13 +266,19 @@ static int devfreq_cooling_state2power(struct thermal_cooling_device *cdev, unsigned long state, u32 *power) { struct devfreq_cooling_device *dfc = cdev->devdata; + struct em_perf_state *table; int perf_idx; if (state > dfc->max_state) return -EINVAL; perf_idx = dfc->max_state - state; - *power = dfc->em_pd->table[perf_idx].power; + + rcu_read_lock(); + table = em_perf_state_from_pd(dfc->em_pd); + *power = table[perf_idx].power; + rcu_read_unlock(); + *power /= MICROWATT_PER_MILLIWATT; return 0; @@ -264,6 +291,7 @@ static int devfreq_cooling_power2state(struct thermal_cooling_device *cdev, struct devfreq *df = dfc->devfreq; struct devfreq_dev_status status; unsigned long freq, em_power_mw; + struct em_perf_state *table; s32 est_power; int i; @@ -288,13 +316,16 @@ static int devfreq_cooling_power2state(struct thermal_cooling_device *cdev, * Find the first cooling state that is within the power * budget. The EM power table is sorted ascending. */ + rcu_read_lock(); + table = em_perf_state_from_pd(dfc->em_pd); for (i = dfc->max_state; i > 0; i--) { /* Convert EM power to milli-Watts to make safe comparison */ - em_power_mw = dfc->em_pd->table[i].power; + em_power_mw = table[i].power; em_power_mw /= MICROWATT_PER_MILLIWATT; if (est_power >= em_power_mw) break; } + rcu_read_unlock(); *state = dfc->max_state - i; dfc->capped_state = *state; diff --git a/drivers/thermal/gov_bang_bang.c b/drivers/thermal/gov_bang_bang.c index 6ddf0accdc98..c3b2943a2db8 100644 --- a/drivers/thermal/gov_bang_bang.c +++ b/drivers/thermal/gov_bang_bang.c @@ -49,7 +49,7 @@ static int thermal_zone_trip_update(struct thermal_zone_device *tz, if (instance->target == 0 && tz->temperature >= trip->temperature) instance->target = 1; else if (instance->target == 1 && - tz->temperature <= trip->temperature - trip->hysteresis) + tz->temperature < trip->temperature - trip->hysteresis) instance->target = 0; dev_dbg(&instance->cdev->device, "target=%d\n", diff --git a/drivers/thermal/gov_fair_share.c b/drivers/thermal/gov_fair_share.c index 538abb7de4e2..4da25a0009d7 100644 --- a/drivers/thermal/gov_fair_share.c +++ b/drivers/thermal/gov_fair_share.c @@ -18,22 +18,24 @@ static int get_trip_level(struct thermal_zone_device *tz) { const struct thermal_trip *trip, *level_trip = NULL; - int trip_level; + int trip_level = -1; for_each_trip(tz, trip) { if (trip->temperature >= tz->temperature) - break; + continue; - level_trip = trip; + trip_level++; + + if (!level_trip || trip->temperature > level_trip->temperature) + level_trip = trip; } /* Bail out if the temperature is not greater than any trips. */ - if (!level_trip) + if (trip_level < 0) return 0; - trip_level = thermal_zone_trip_id(tz, level_trip); - - trace_thermal_zone_trip(tz, trip_level, level_trip->type); + trace_thermal_zone_trip(tz, thermal_zone_trip_id(tz, level_trip), + level_trip->type); return trip_level; } diff --git a/drivers/thermal/gov_power_allocator.c b/drivers/thermal/gov_power_allocator.c index 81e061f183ad..1b17dc4c219c 100644 --- a/drivers/thermal/gov_power_allocator.c +++ b/drivers/thermal/gov_power_allocator.c @@ -711,6 +711,8 @@ static int power_allocator_bind(struct thermal_zone_device *tz) if (!tz->tzp->sustainable_power) dev_warn(&tz->device, "power_allocator: sustainable_power will be estimated\n"); + else + params->sustainable_power = tz->tzp->sustainable_power; estimate_pid_constants(tz, tz->tzp->sustainable_power, params->trip_switch_on, diff --git a/drivers/thermal/imx_thermal.c b/drivers/thermal/imx_thermal.c index 7019c4fdd549..83eaae5ca3b8 100644 --- a/drivers/thermal/imx_thermal.c +++ b/drivers/thermal/imx_thermal.c @@ -115,7 +115,8 @@ struct thermal_soc_data { }; static struct thermal_trip trips[] = { - [IMX_TRIP_PASSIVE] = { .type = THERMAL_TRIP_PASSIVE }, + [IMX_TRIP_PASSIVE] = { .type = THERMAL_TRIP_PASSIVE, + .flags = THERMAL_TRIP_FLAG_RW_TEMP }, [IMX_TRIP_CRITICAL] = { .type = THERMAL_TRIP_CRITICAL }, }; @@ -354,6 +355,7 @@ static int imx_set_trip_temp(struct thermal_zone_device *tz, int trip_id, return -EINVAL; imx_set_alarm_temp(data, temp); + trips[IMX_TRIP_PASSIVE].temperature = temp; pm_runtime_put(data->dev); @@ -699,7 +701,7 @@ static int imx_thermal_probe(struct platform_device *pdev) data->tz = thermal_zone_device_register_with_trips("imx_thermal_zone", trips, ARRAY_SIZE(trips), - BIT(IMX_TRIP_PASSIVE), data, + data, &imx_tz_ops, NULL, IMX_PASSIVE_DELAY, IMX_POLLING_DELAY); diff --git a/drivers/thermal/intel/Kconfig b/drivers/thermal/intel/Kconfig index b43953b5539f..a31f2f32996a 100644 --- a/drivers/thermal/intel/Kconfig +++ b/drivers/thermal/intel/Kconfig @@ -23,7 +23,6 @@ config X86_PKG_TEMP_THERMAL tristate "X86 package temperature thermal driver" depends on X86_THERMAL_VECTOR select THERMAL_GOV_USER_SPACE - select THERMAL_WRITABLE_TRIPS select INTEL_TCC default m help @@ -47,7 +46,6 @@ config INTEL_SOC_DTS_THERMAL tristate "Intel SoCs DTS thermal driver" depends on X86 && PCI && ACPI select INTEL_SOC_DTS_IOSF_CORE - select THERMAL_WRITABLE_TRIPS help Enable this to register Intel SoCs (e.g. Bay Trail) platform digital temperature sensor (DTS). These SoCs have two additional DTSs in diff --git a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c index 3e4bfe817fac..400fde7cb3b1 100644 --- a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c +++ b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.c @@ -58,15 +58,10 @@ static int int340x_thermal_set_trip_temp(struct thermal_zone_device *zone, static void int340x_thermal_critical(struct thermal_zone_device *zone) { - dev_dbg(&zone->device, "%s: critical temperature reached\n", zone->type); + dev_dbg(thermal_zone_device(zone), "%s: critical temperature reached\n", + thermal_zone_device_type(zone)); } -static struct thermal_zone_device_ops int340x_thermal_zone_ops = { - .get_temp = int340x_thermal_get_zone_temp, - .set_trip_temp = int340x_thermal_set_trip_temp, - .critical = int340x_thermal_critical, -}; - static inline void *int_to_trip_priv(int i) { return (void *)(long)i; @@ -126,11 +121,15 @@ static struct thermal_zone_params int340x_thermal_params = { struct int34x_thermal_zone *int340x_thermal_zone_add(struct acpi_device *adev, int (*get_temp) (struct thermal_zone_device *, int *)) { + const struct thermal_zone_device_ops zone_ops = { + .set_trip_temp = int340x_thermal_set_trip_temp, + .critical = int340x_thermal_critical, + .get_temp = get_temp ? get_temp : int340x_thermal_get_zone_temp, + }; struct int34x_thermal_zone *int34x_zone; struct thermal_trip *zone_trips; unsigned long long trip_cnt = 0; unsigned long long hyst; - int trip_mask = 0; acpi_status status; int i, ret; @@ -140,21 +139,9 @@ struct int34x_thermal_zone *int340x_thermal_zone_add(struct acpi_device *adev, int34x_zone->adev = adev; - int34x_zone->ops = kmemdup(&int340x_thermal_zone_ops, - sizeof(int340x_thermal_zone_ops), GFP_KERNEL); - if (!int34x_zone->ops) { - ret = -ENOMEM; - goto err_ops_alloc; - } - - if (get_temp) - int34x_zone->ops->get_temp = get_temp; - status = acpi_evaluate_integer(adev->handle, "PATC", NULL, &trip_cnt); - if (ACPI_SUCCESS(status)) { + if (ACPI_SUCCESS(status)) int34x_zone->aux_trip_nr = trip_cnt; - trip_mask = BIT(trip_cnt) - 1; - } zone_trips = kzalloc(sizeof(*zone_trips) * (trip_cnt + INT340X_THERMAL_MAX_TRIP_COUNT), GFP_KERNEL); @@ -166,6 +153,7 @@ struct int34x_thermal_zone *int340x_thermal_zone_add(struct acpi_device *adev, for (i = 0; i < trip_cnt; i++) { zone_trips[i].type = THERMAL_TRIP_PASSIVE; zone_trips[i].temperature = THERMAL_TEMP_INVALID; + zone_trips[i].flags |= THERMAL_TRIP_FLAG_RW_TEMP; } trip_cnt = int340x_thermal_read_trips(adev, zone_trips, trip_cnt); @@ -179,17 +167,17 @@ struct int34x_thermal_zone *int340x_thermal_zone_add(struct acpi_device *adev, for (i = 0; i < trip_cnt; ++i) zone_trips[i].hysteresis = hyst; - int34x_zone->trips = zone_trips; - int34x_zone->lpat_table = acpi_lpat_get_conversion_table(adev->handle); int34x_zone->zone = thermal_zone_device_register_with_trips( acpi_device_bid(adev), zone_trips, trip_cnt, - trip_mask, int34x_zone, - int34x_zone->ops, + int34x_zone, + &zone_ops, &int340x_thermal_params, 0, 0); + kfree(zone_trips); + if (IS_ERR(int34x_zone->zone)) { ret = PTR_ERR(int34x_zone->zone); goto err_thermal_zone; @@ -203,11 +191,8 @@ struct int34x_thermal_zone *int340x_thermal_zone_add(struct acpi_device *adev, err_enable: thermal_zone_device_unregister(int34x_zone->zone); err_thermal_zone: - kfree(int34x_zone->trips); acpi_lpat_free_conversion_table(int34x_zone->lpat_table); err_trips_alloc: - kfree(int34x_zone->ops); -err_ops_alloc: kfree(int34x_zone); return ERR_PTR(ret); } @@ -217,8 +202,6 @@ void int340x_thermal_zone_remove(struct int34x_thermal_zone *int34x_zone) { thermal_zone_device_unregister(int34x_zone->zone); acpi_lpat_free_conversion_table(int34x_zone->lpat_table); - kfree(int34x_zone->trips); - kfree(int34x_zone->ops); kfree(int34x_zone); } EXPORT_SYMBOL_GPL(int340x_thermal_zone_remove); diff --git a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.h b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.h index e0df6271facc..d504e271009a 100644 --- a/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.h +++ b/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.h @@ -20,10 +20,8 @@ struct active_trip { struct int34x_thermal_zone { struct acpi_device *adev; - struct thermal_trip *trips; int aux_trip_nr; struct thermal_zone_device *zone; - struct thermal_zone_device_ops *ops; void *priv_data; struct acpi_lpat_conversion_table *lpat_table; }; diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c index 649f67fdf345..d75fae7b7ed2 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c @@ -176,14 +176,14 @@ static int proc_thermal_get_zone_temp(struct thermal_zone_device *zone, int *temp) { int cpu; - int curr_temp; + int curr_temp, ret; *temp = 0; for_each_online_cpu(cpu) { - curr_temp = intel_tcc_get_temp(cpu, false); - if (curr_temp < 0) - return curr_temp; + ret = intel_tcc_get_temp(cpu, &curr_temp, false); + if (ret < 0) + return ret; if (!*temp || curr_temp > *temp) *temp = curr_temp; } diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h index 95c6013a33fb..674f3c85dfbc 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h @@ -25,6 +25,7 @@ #define PCI_DEVICE_ID_INTEL_HSB_THERMAL 0x0A03 #define PCI_DEVICE_ID_INTEL_ICL_THERMAL 0x8a03 #define PCI_DEVICE_ID_INTEL_JSL_THERMAL 0x4E03 +#define PCI_DEVICE_ID_INTEL_LNLM_THERMAL 0x641D #define PCI_DEVICE_ID_INTEL_MTLP_THERMAL 0x7D03 #define PCI_DEVICE_ID_INTEL_RPL_THERMAL 0xA71D #define PCI_DEVICE_ID_INTEL_SKL_THERMAL 0x1903 diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c index d7495571dd5d..14e34eabc419 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c @@ -233,11 +233,7 @@ static int get_trip_temp(struct proc_thermal_pci *pci_info) return temp; } -static struct thermal_trip psv_trip = { - .type = THERMAL_TRIP_PASSIVE, -}; - -static struct thermal_zone_device_ops tzone_ops = { +static const struct thermal_zone_device_ops tzone_ops = { .get_temp = sys_get_curr_temp, .set_trip_temp = sys_set_trip_temp, }; @@ -251,6 +247,10 @@ static int proc_thermal_pci_probe(struct pci_dev *pdev, const struct pci_device_ { struct proc_thermal_device *proc_priv; struct proc_thermal_pci *pci_info; + struct thermal_trip psv_trip = { + .type = THERMAL_TRIP_PASSIVE, + .flags = THERMAL_TRIP_FLAG_RW_TEMP, + }; int irq_flag = 0, irq, ret; bool msi_irq = false; @@ -290,7 +290,7 @@ static int proc_thermal_pci_probe(struct pci_dev *pdev, const struct pci_device_ psv_trip.temperature = get_trip_temp(pci_info); pci_info->tzone = thermal_zone_device_register_with_trips("TCPU_PCI", &psv_trip, - 1, 1, pci_info, + 1, pci_info, &tzone_ops, &tzone_params, 0, 0); if (IS_ERR(pci_info->tzone)) { @@ -407,6 +407,7 @@ static SIMPLE_DEV_PM_OPS(proc_thermal_pci_pm, proc_thermal_pci_suspend, static const struct pci_device_id proc_thermal_pci_ids[] = { { PCI_DEVICE_DATA(INTEL, ADL_THERMAL, PROC_THERMAL_FEATURE_RAPL | PROC_THERMAL_FEATURE_FIVR | PROC_THERMAL_FEATURE_DVFS | PROC_THERMAL_FEATURE_WT_REQ) }, + { PCI_DEVICE_DATA(INTEL, LNLM_THERMAL, PROC_THERMAL_FEATURE_RAPL) }, { PCI_DEVICE_DATA(INTEL, MTLP_THERMAL, PROC_THERMAL_FEATURE_RAPL | PROC_THERMAL_FEATURE_FIVR | PROC_THERMAL_FEATURE_DVFS | PROC_THERMAL_FEATURE_DLVR | PROC_THERMAL_FEATURE_WT_HINT | PROC_THERMAL_FEATURE_POWER_FLOOR) }, diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c index 2f00fc3bf274..e964a9375722 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c @@ -27,9 +27,9 @@ static int rapl_mmio_cpu_online(unsigned int cpu) if (topology_physical_package_id(cpu)) return 0; - rp = rapl_find_package_domain(cpu, &rapl_mmio_priv, true); + rp = rapl_find_package_domain_cpuslocked(cpu, &rapl_mmio_priv, true); if (!rp) { - rp = rapl_add_package(cpu, &rapl_mmio_priv, true); + rp = rapl_add_package_cpuslocked(cpu, &rapl_mmio_priv, true); if (IS_ERR(rp)) return PTR_ERR(rp); } @@ -42,14 +42,14 @@ static int rapl_mmio_cpu_down_prep(unsigned int cpu) struct rapl_package *rp; int lead_cpu; - rp = rapl_find_package_domain(cpu, &rapl_mmio_priv, true); + rp = rapl_find_package_domain_cpuslocked(cpu, &rapl_mmio_priv, true); if (!rp) return 0; cpumask_clear_cpu(cpu, &rp->cpumask); lead_cpu = cpumask_first(&rp->cpumask); if (lead_cpu >= nr_cpu_ids) - rapl_remove_package(rp); + rapl_remove_package_cpuslocked(rp); else if (rp->lead_cpu == cpu) rp->lead_cpu = lead_cpu; return 0; diff --git a/drivers/thermal/intel/intel_pch_thermal.c b/drivers/thermal/intel/intel_pch_thermal.c index b3905e34c507..f5be2c389351 100644 --- a/drivers/thermal/intel/intel_pch_thermal.c +++ b/drivers/thermal/intel/intel_pch_thermal.c @@ -84,7 +84,6 @@ struct pch_thermal_device { void __iomem *hw_base; struct pci_dev *pdev; struct thermal_zone_device *tzd; - struct thermal_trip trips[PCH_MAX_TRIPS]; bool bios_enabled; }; @@ -94,7 +93,8 @@ struct pch_thermal_device { * passive trip temperature using _PSV method. There is no specific * passive temperature setting in MMIO interface of this PCI device. */ -static int pch_wpt_add_acpi_psv_trip(struct pch_thermal_device *ptd, int trip) +static int pch_wpt_add_acpi_psv_trip(struct pch_thermal_device *ptd, + struct thermal_trip *trip) { struct acpi_device *adev; int temp; @@ -106,12 +106,13 @@ static int pch_wpt_add_acpi_psv_trip(struct pch_thermal_device *ptd, int trip) if (thermal_acpi_passive_trip_temp(adev, &temp) || temp <= 0) return 0; - ptd->trips[trip].type = THERMAL_TRIP_PASSIVE; - ptd->trips[trip].temperature = temp; + trip->type = THERMAL_TRIP_PASSIVE; + trip->temperature = temp; return 1; } #else -static int pch_wpt_add_acpi_psv_trip(struct pch_thermal_device *ptd, int trip) +static int pch_wpt_add_acpi_psv_trip(struct pch_thermal_device *ptd, + struct thermal_trip *trip) { return 0; } @@ -131,7 +132,7 @@ static void pch_critical(struct thermal_zone_device *tzd) thermal_zone_device_type(tzd)); } -static struct thermal_zone_device_ops tzd_ops = { +static const struct thermal_zone_device_ops tzd_ops = { .get_temp = pch_thermal_get_temp, .critical = pch_critical, }; @@ -159,6 +160,7 @@ static const char *board_names[] = { static int intel_pch_thermal_probe(struct pci_dev *pdev, const struct pci_device_id *id) { + struct thermal_trip ptd_trips[PCH_MAX_TRIPS] = { 0 }; enum pch_board_ids board_id = id->driver_data; struct pch_thermal_device *ptd; int nr_trips = 0; @@ -220,22 +222,22 @@ read_trips: trip_temp = readw(ptd->hw_base + WPT_CTT); trip_temp &= 0x1FF; if (trip_temp) { - ptd->trips[nr_trips].temperature = GET_WPT_TEMP(trip_temp); - ptd->trips[nr_trips++].type = THERMAL_TRIP_CRITICAL; + ptd_trips[nr_trips].temperature = GET_WPT_TEMP(trip_temp); + ptd_trips[nr_trips++].type = THERMAL_TRIP_CRITICAL; } trip_temp = readw(ptd->hw_base + WPT_PHL); trip_temp &= 0x1FF; if (trip_temp) { - ptd->trips[nr_trips].temperature = GET_WPT_TEMP(trip_temp); - ptd->trips[nr_trips++].type = THERMAL_TRIP_HOT; + ptd_trips[nr_trips].temperature = GET_WPT_TEMP(trip_temp); + ptd_trips[nr_trips++].type = THERMAL_TRIP_HOT; } - nr_trips += pch_wpt_add_acpi_psv_trip(ptd, nr_trips); + nr_trips += pch_wpt_add_acpi_psv_trip(ptd, &ptd_trips[nr_trips]); ptd->tzd = thermal_zone_device_register_with_trips(board_names[board_id], - ptd->trips, nr_trips, - 0, ptd, &tzd_ops, + ptd_trips, nr_trips, + ptd, &tzd_ops, NULL, 0, 0); if (IS_ERR(ptd->tzd)) { dev_err(&pdev->dev, "Failed to register thermal zone %s\n", diff --git a/drivers/thermal/intel/intel_quark_dts_thermal.c b/drivers/thermal/intel/intel_quark_dts_thermal.c index 646ca8bd40a9..ec6ad26027bc 100644 --- a/drivers/thermal/intel/intel_quark_dts_thermal.c +++ b/drivers/thermal/intel/intel_quark_dts_thermal.c @@ -93,10 +93,6 @@ /* Quark DTS has 2 trip points: hot & catastrophic */ #define QRK_MAX_DTS_TRIPS 2 -/* If DTS not locked, all trip points are configurable */ -#define QRK_DTS_WR_MASK_SET 0x3 -/* If DTS locked, all trip points are not configurable */ -#define QRK_DTS_WR_MASK_CLR 0 #define DEFAULT_POLL_DELAY 2000 @@ -105,7 +101,6 @@ struct soc_sensor_entry { u32 store_ptps; u32 store_dts_enable; struct thermal_zone_device *tzone; - struct thermal_trip trips[QRK_MAX_DTS_TRIPS]; }; static struct soc_sensor_entry *soc_dts; @@ -293,7 +288,7 @@ static int sys_change_mode(struct thermal_zone_device *tzd, return ret; } -static struct thermal_zone_device_ops tzone_ops = { +static const struct thermal_zone_device_ops tzone_ops = { .get_temp = sys_get_curr_temp, .set_trip_temp = sys_set_trip_temp, .change_mode = sys_change_mode, @@ -320,10 +315,10 @@ static void free_soc_dts(struct soc_sensor_entry *aux_entry) static struct soc_sensor_entry *alloc_soc_dts(void) { + struct thermal_trip trips[QRK_MAX_DTS_TRIPS] = { 0 }; struct soc_sensor_entry *aux_entry; int err; u32 out; - int wr_mask; aux_entry = kzalloc(sizeof(*aux_entry), GFP_KERNEL); if (!aux_entry) { @@ -337,13 +332,7 @@ static struct soc_sensor_entry *alloc_soc_dts(void) if (err) goto err_ret; - if (out & QRK_DTS_LOCK_BIT) { - aux_entry->locked = true; - wr_mask = QRK_DTS_WR_MASK_CLR; - } else { - aux_entry->locked = false; - wr_mask = QRK_DTS_WR_MASK_SET; - } + aux_entry->locked = !!(out & QRK_DTS_LOCK_BIT); /* Store DTS default state if DTS registers are not locked */ if (!aux_entry->locked) { @@ -360,19 +349,22 @@ static struct soc_sensor_entry *alloc_soc_dts(void) &aux_entry->store_ptps); if (err) goto err_ret; + + trips[QRK_DTS_ID_TP_CRITICAL].flags |= THERMAL_TRIP_FLAG_RW_TEMP; + trips[QRK_DTS_ID_TP_HOT].flags |= THERMAL_TRIP_FLAG_RW_TEMP; } - aux_entry->trips[QRK_DTS_ID_TP_CRITICAL].temperature = get_trip_temp(QRK_DTS_ID_TP_CRITICAL); - aux_entry->trips[QRK_DTS_ID_TP_CRITICAL].type = THERMAL_TRIP_CRITICAL; + trips[QRK_DTS_ID_TP_CRITICAL].temperature = get_trip_temp(QRK_DTS_ID_TP_CRITICAL); + trips[QRK_DTS_ID_TP_CRITICAL].type = THERMAL_TRIP_CRITICAL; - aux_entry->trips[QRK_DTS_ID_TP_HOT].temperature = get_trip_temp(QRK_DTS_ID_TP_HOT); - aux_entry->trips[QRK_DTS_ID_TP_HOT].type = THERMAL_TRIP_HOT; + trips[QRK_DTS_ID_TP_HOT].temperature = get_trip_temp(QRK_DTS_ID_TP_HOT); + trips[QRK_DTS_ID_TP_HOT].type = THERMAL_TRIP_HOT; aux_entry->tzone = thermal_zone_device_register_with_trips("quark_dts", - aux_entry->trips, + trips, QRK_MAX_DTS_TRIPS, - wr_mask, - aux_entry, &tzone_ops, + aux_entry, + &tzone_ops, NULL, 0, polling_delay); if (IS_ERR(aux_entry->tzone)) { err = PTR_ERR(aux_entry->tzone); diff --git a/drivers/thermal/intel/intel_soc_dts_iosf.c b/drivers/thermal/intel/intel_soc_dts_iosf.c index d00def3c4703..2ab943b66f7a 100644 --- a/drivers/thermal/intel/intel_soc_dts_iosf.c +++ b/drivers/thermal/intel/intel_soc_dts_iosf.c @@ -129,22 +129,6 @@ err_restore_ptps: return status; } -static int configure_trip(struct intel_soc_dts_sensor_entry *dts, - int thres_index, enum thermal_trip_type trip_type, - int temp) -{ - int ret; - - ret = update_trip_temp(dts->sensors, thres_index, temp); - if (ret) - return ret; - - dts->trips[thres_index].temperature = temp; - dts->trips[thres_index].type = trip_type; - - return 0; -} - static int sys_set_trip_temp(struct thermal_zone_device *tzd, int trip, int temp) { @@ -184,7 +168,7 @@ static int sys_get_curr_temp(struct thermal_zone_device *tzd, return 0; } -static struct thermal_zone_device_ops tzone_ops = { +static const struct thermal_zone_device_ops tzone_ops = { .get_temp = sys_get_curr_temp, .set_trip_temp = sys_set_trip_temp, }; @@ -218,15 +202,10 @@ static void remove_dts_thermal_zone(struct intel_soc_dts_sensor_entry *dts) } static int add_dts_thermal_zone(int id, struct intel_soc_dts_sensor_entry *dts, - bool critical_trip) + struct thermal_trip *trips) { - int writable_trip_cnt = SOC_MAX_DTS_TRIPS; char name[10]; - unsigned long trip; - int trip_mask; - unsigned long ptps; u32 store_ptps; - unsigned long i; int ret; /* Store status to restor on exit */ @@ -237,26 +216,20 @@ static int add_dts_thermal_zone(int id, struct intel_soc_dts_sensor_entry *dts, dts->id = id; - if (critical_trip) - writable_trip_cnt--; - - trip_mask = GENMASK(writable_trip_cnt - 1, 0); - /* Check if the writable trip we provide is not used by BIOS */ ret = iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_REG_READ, SOC_DTS_OFFSET_PTPS, &store_ptps); - if (ret) - trip_mask = 0; - else { - ptps = store_ptps; - for_each_set_clump8(i, trip, &ptps, writable_trip_cnt * 8) - trip_mask &= ~BIT(i / 8); + if (!ret) { + int i; + + for (i = 0; i <= 1; i++) { + if (store_ptps & (0xFFU << i * 8)) + trips[i].flags &= ~THERMAL_TRIP_FLAG_RW_TEMP; + } } - dts->trip_mask = trip_mask; snprintf(name, sizeof(name), "soc_dts%d", id); - dts->tzone = thermal_zone_device_register_with_trips(name, dts->trips, + dts->tzone = thermal_zone_device_register_with_trips(name, trips, SOC_MAX_DTS_TRIPS, - trip_mask, dts, &tzone_ops, NULL, 0, 0); if (IS_ERR(dts->tzone)) { @@ -315,14 +288,23 @@ EXPORT_SYMBOL_GPL(intel_soc_dts_iosf_interrupt_handler); static void dts_trips_reset(struct intel_soc_dts_sensors *sensors, int dts_index) { - configure_trip(&sensors->soc_dts[dts_index], 0, 0, 0); - configure_trip(&sensors->soc_dts[dts_index], 1, 0, 0); + update_trip_temp(sensors, 0, 0); + update_trip_temp(sensors, 1, 0); +} + +static void set_trip(struct thermal_trip *trip, enum thermal_trip_type type, + u8 flags, int temp) +{ + trip->type = type; + trip->flags = flags; + trip->temperature = temp; } struct intel_soc_dts_sensors * intel_soc_dts_iosf_init(enum intel_soc_dts_interrupt_type intr_type, bool critical_trip, int crit_offset) { + struct thermal_trip trips[SOC_MAX_DTS_SENSORS][SOC_MAX_DTS_TRIPS] = { 0 }; struct intel_soc_dts_sensors *sensors; int tj_max; int ret; @@ -345,30 +327,33 @@ intel_soc_dts_iosf_init(enum intel_soc_dts_interrupt_type intr_type, sensors->tj_max = tj_max * 1000; for (i = 0; i < SOC_MAX_DTS_SENSORS; ++i) { - enum thermal_trip_type trip_type; int temp; sensors->soc_dts[i].sensors = sensors; - ret = configure_trip(&sensors->soc_dts[i], 0, - THERMAL_TRIP_PASSIVE, 0); + set_trip(&trips[i][0], THERMAL_TRIP_PASSIVE, + THERMAL_TRIP_FLAG_RW_TEMP, 0); + + ret = update_trip_temp(sensors, 0, 0); if (ret) goto err_reset_trips; if (critical_trip) { - trip_type = THERMAL_TRIP_CRITICAL; temp = sensors->tj_max - crit_offset; + set_trip(&trips[i][1], THERMAL_TRIP_CRITICAL, 0, temp); } else { - trip_type = THERMAL_TRIP_PASSIVE; + set_trip(&trips[i][1], THERMAL_TRIP_PASSIVE, + THERMAL_TRIP_FLAG_RW_TEMP, 0); temp = 0; } - ret = configure_trip(&sensors->soc_dts[i], 1, trip_type, temp); + + ret = update_trip_temp(sensors, 1, temp); if (ret) goto err_reset_trips; } for (i = 0; i < SOC_MAX_DTS_SENSORS; ++i) { - ret = add_dts_thermal_zone(i, &sensors->soc_dts[i], critical_trip); + ret = add_dts_thermal_zone(i, &sensors->soc_dts[i], trips[i]); if (ret) goto err_remove_zone; } diff --git a/drivers/thermal/intel/intel_soc_dts_iosf.h b/drivers/thermal/intel/intel_soc_dts_iosf.h index 162841df0ebe..44eee844ab3c 100644 --- a/drivers/thermal/intel/intel_soc_dts_iosf.h +++ b/drivers/thermal/intel/intel_soc_dts_iosf.h @@ -28,8 +28,6 @@ struct intel_soc_dts_sensors; struct intel_soc_dts_sensor_entry { int id; u32 store_status; - u32 trip_mask; - struct thermal_trip trips[SOC_MAX_DTS_TRIPS]; struct thermal_zone_device *tzone; struct intel_soc_dts_sensors *sensors; }; diff --git a/drivers/thermal/intel/intel_tcc.c b/drivers/thermal/intel/intel_tcc.c index 2e5c741c41ca..5e8b7f34b395 100644 --- a/drivers/thermal/intel/intel_tcc.c +++ b/drivers/thermal/intel/intel_tcc.c @@ -103,18 +103,19 @@ EXPORT_SYMBOL_NS_GPL(intel_tcc_set_offset, INTEL_TCC); /** * intel_tcc_get_temp() - returns the current temperature * @cpu: cpu that the MSR should be run on, nagative value means any cpu. + * @temp: pointer to the memory for saving cpu temperature. * @pkg: true: Package Thermal Sensor. false: Core Thermal Sensor. * * Get the current temperature returned by the CPU core/package level * thermal sensor, in degrees C. * - * Return: Temperature in degrees C on success, negative error code otherwise. + * Return: 0 on success, negative error code otherwise. */ -int intel_tcc_get_temp(int cpu, bool pkg) +int intel_tcc_get_temp(int cpu, int *temp, bool pkg) { u32 low, high; u32 msr = pkg ? MSR_IA32_PACKAGE_THERM_STATUS : MSR_IA32_THERM_STATUS; - int tjmax, temp, err; + int tjmax, err; tjmax = intel_tcc_get_tjmax(cpu); if (tjmax < 0) @@ -131,9 +132,8 @@ int intel_tcc_get_temp(int cpu, bool pkg) if (!(low & BIT(31))) return -ENODATA; - temp = tjmax - ((low >> 16) & 0x7f); + *temp = tjmax - ((low >> 16) & 0x7f); - /* Do not allow negative CPU temperature */ - return temp >= 0 ? temp : -ENODATA; + return 0; } EXPORT_SYMBOL_NS_GPL(intel_tcc_get_temp, INTEL_TCC); diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c index f6c2e5964b8f..c0ca8e3ff2e7 100644 --- a/drivers/thermal/intel/x86_pkg_temp_thermal.c +++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c @@ -53,7 +53,6 @@ struct zone_device { u32 msr_pkg_therm_high; struct delayed_work work; struct thermal_zone_device *tzone; - struct thermal_trip *trips; struct cpumask cpumask; }; @@ -108,11 +107,11 @@ static struct zone_device *pkg_temp_thermal_get_dev(unsigned int cpu) static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp) { struct zone_device *zonedev = thermal_zone_device_priv(tzd); - int val; + int val, ret; - val = intel_tcc_get_temp(zonedev->cpu, true); - if (val < 0) - return val; + ret = intel_tcc_get_temp(zonedev->cpu, &val, true); + if (ret < 0) + return ret; *temp = val * 1000; pr_debug("sys_get_curr_temp %d\n", *temp); @@ -167,7 +166,7 @@ sys_set_trip_temp(struct thermal_zone_device *tzd, int trip, int temp) } /* Thermal zone callback registry */ -static struct thermal_zone_device_ops tzone_ops = { +static const struct thermal_zone_device_ops tzone_ops = { .get_temp = sys_get_curr_temp, .set_trip_temp = sys_set_trip_temp, }; @@ -268,17 +267,13 @@ static int pkg_thermal_notify(u64 msr_val) return 0; } -static struct thermal_trip *pkg_temp_thermal_trips_init(int cpu, int tj_max, int num_trips) +static int pkg_temp_thermal_trips_init(int cpu, int tj_max, + struct thermal_trip *trips, int num_trips) { - struct thermal_trip *trips; unsigned long thres_reg_value; u32 mask, shift, eax, edx; int ret, i; - trips = kzalloc(sizeof(*trips) * num_trips, GFP_KERNEL); - if (!trips) - return ERR_PTR(-ENOMEM); - for (i = 0; i < num_trips; i++) { if (i) { @@ -291,10 +286,8 @@ static struct thermal_trip *pkg_temp_thermal_trips_init(int cpu, int tj_max, int ret = rdmsr_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, &eax, &edx); - if (ret < 0) { - kfree(trips); - return ERR_PTR(ret); - } + if (ret < 0) + return ret; thres_reg_value = (eax & mask) >> shift; @@ -302,16 +295,18 @@ static struct thermal_trip *pkg_temp_thermal_trips_init(int cpu, int tj_max, int tj_max - thres_reg_value * 1000 : THERMAL_TEMP_INVALID; trips[i].type = THERMAL_TRIP_PASSIVE; + trips[i].flags |= THERMAL_TRIP_FLAG_RW_TEMP; pr_debug("%s: cpu=%d, trip=%d, temp=%d\n", __func__, cpu, i, trips[i].temperature); } - return trips; + return 0; } static int pkg_temp_thermal_device_add(unsigned int cpu) { + struct thermal_trip trips[MAX_NUMBER_OF_TRIPS] = { 0 }; int id = topology_logical_die_id(cpu); u32 eax, ebx, ecx, edx; struct zone_device *zonedev; @@ -336,21 +331,18 @@ static int pkg_temp_thermal_device_add(unsigned int cpu) if (!zonedev) return -ENOMEM; - zonedev->trips = pkg_temp_thermal_trips_init(cpu, tj_max, thres_count); - if (IS_ERR(zonedev->trips)) { - err = PTR_ERR(zonedev->trips); + err = pkg_temp_thermal_trips_init(cpu, tj_max, trips, thres_count); + if (err) goto out_kfree_zonedev; - } INIT_DELAYED_WORK(&zonedev->work, pkg_temp_thermal_threshold_work_fn); zonedev->cpu = cpu; zonedev->tzone = thermal_zone_device_register_with_trips("x86_pkg_temp", - zonedev->trips, thres_count, - (thres_count == MAX_NUMBER_OF_TRIPS) ? 0x03 : 0x01, + trips, thres_count, zonedev, &tzone_ops, &pkg_temp_tz_params, 0, 0); if (IS_ERR(zonedev->tzone)) { err = PTR_ERR(zonedev->tzone); - goto out_kfree_trips; + goto out_kfree_zonedev; } err = thermal_zone_device_enable(zonedev->tzone); if (err) @@ -369,8 +361,6 @@ static int pkg_temp_thermal_device_add(unsigned int cpu) out_unregister_tz: thermal_zone_device_unregister(zonedev->tzone); -out_kfree_trips: - kfree(zonedev->trips); out_kfree_zonedev: kfree(zonedev); return err; @@ -457,10 +447,9 @@ static int pkg_thermal_cpu_offline(unsigned int cpu) raw_spin_unlock_irq(&pkg_temp_lock); /* Final cleanup if this is the last cpu */ - if (lastcpu) { - kfree(zonedev->trips); + if (lastcpu) kfree(zonedev); - } + return 0; } diff --git a/drivers/thermal/rcar_thermal.c b/drivers/thermal/rcar_thermal.c index feb848d595fa..925183753fcb 100644 --- a/drivers/thermal/rcar_thermal.c +++ b/drivers/thermal/rcar_thermal.c @@ -489,7 +489,7 @@ static int rcar_thermal_probe(struct platform_device *pdev) &rcar_thermal_zone_ops); } else { priv->zone = thermal_zone_device_register_with_trips( - "rcar_thermal", trips, ARRAY_SIZE(trips), 0, priv, + "rcar_thermal", trips, ARRAY_SIZE(trips), priv, &rcar_thermal_zone_ops, NULL, 0, idle); diff --git a/drivers/thermal/st/st_thermal.c b/drivers/thermal/st/st_thermal.c index 0d6249b36609..2a105409864e 100644 --- a/drivers/thermal/st/st_thermal.c +++ b/drivers/thermal/st/st_thermal.c @@ -203,7 +203,7 @@ int st_thermal_register(struct platform_device *pdev, trip.type = THERMAL_TRIP_CRITICAL; sensor->thermal_dev = - thermal_zone_device_register_with_trips(dev_name(dev), &trip, 1, 0, sensor, + thermal_zone_device_register_with_trips(dev_name(dev), &trip, 1, sensor, &st_tz_ops, NULL, 0, polling_delay); if (IS_ERR(sensor->thermal_dev)) { dev_err(dev, "failed to register thermal zone device\n"); diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index dfaa6341694a..34a31bc72023 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -273,7 +273,6 @@ static int __init thermal_register_governors(void) /* * Zone update section: main control loop applied to each zone while monitoring - * * in polling mode. The monitoring is done using a workqueue. * Same update may be done on a zone by calling thermal_zone_device_update(). * @@ -356,9 +355,9 @@ static void handle_critical_trips(struct thermal_zone_device *tz, trace_thermal_zone_trip(tz, thermal_zone_trip_id(tz, trip), trip->type); if (trip->type == THERMAL_TRIP_CRITICAL) - tz->ops->critical(tz); - else if (tz->ops->hot) - tz->ops->hot(tz); + tz->ops.critical(tz); + else if (tz->ops.hot) + tz->ops.hot(tz); } static void handle_thermal_trip(struct thermal_zone_device *tz, @@ -493,8 +492,8 @@ static int thermal_zone_device_set_mode(struct thermal_zone_device *tz, return ret; } - if (tz->ops->change_mode) - ret = tz->ops->change_mode(tz, mode); + if (tz->ops.change_mode) + ret = tz->ops.change_mode(tz, mode); if (!ret) tz->mode = mode; @@ -867,8 +866,8 @@ static void bind_cdev(struct thermal_cooling_device *cdev) struct thermal_zone_device *pos = NULL; list_for_each_entry(pos, &thermal_tz_list, node) { - if (pos->ops->bind) { - ret = pos->ops->bind(pos, cdev); + if (pos->ops.bind) { + ret = pos->ops.bind(pos, cdev); if (ret) print_bind_err_msg(pos, cdev, ret); } @@ -1184,8 +1183,8 @@ void thermal_cooling_device_unregister(struct thermal_cooling_device *cdev) /* Unbind all thermal zones associated with 'this' cdev */ list_for_each_entry(tz, &thermal_tz_list, node) { - if (tz->ops->unbind) - tz->ops->unbind(tz, cdev); + if (tz->ops.unbind) + tz->ops.unbind(tz, cdev); } mutex_unlock(&thermal_list_lock); @@ -1199,13 +1198,13 @@ static void bind_tz(struct thermal_zone_device *tz) int ret; struct thermal_cooling_device *pos = NULL; - if (!tz->ops->bind) + if (!tz->ops.bind) return; mutex_lock(&thermal_list_lock); list_for_each_entry(pos, &thermal_cdev_list, node) { - ret = tz->ops->bind(tz, pos); + ret = tz->ops.bind(tz, pos); if (ret) print_bind_err_msg(tz, pos, ret); } @@ -1224,11 +1223,8 @@ int thermal_zone_get_crit_temp(struct thermal_zone_device *tz, int *temp) { int i, ret = -EINVAL; - if (tz->ops->get_crit_temp) - return tz->ops->get_crit_temp(tz, temp); - - if (!tz->trips) - return -EINVAL; + if (tz->ops.get_crit_temp) + return tz->ops.get_crit_temp(tz, temp); mutex_lock(&tz->lock); @@ -1251,7 +1247,6 @@ EXPORT_SYMBOL_GPL(thermal_zone_get_crit_temp); * @type: the thermal zone device type * @trips: a pointer to an array of thermal trips * @num_trips: the number of trip points the thermal zone support - * @mask: a bit string indicating the writeablility of trip points * @devdata: private device data * @ops: standard thermal zone device callbacks * @tzp: thermal zone platform parameters @@ -1272,10 +1267,12 @@ EXPORT_SYMBOL_GPL(thermal_zone_get_crit_temp); * IS_ERR*() helpers. */ struct thermal_zone_device * -thermal_zone_device_register_with_trips(const char *type, struct thermal_trip *trips, int num_trips, int mask, - void *devdata, struct thermal_zone_device_ops *ops, - const struct thermal_zone_params *tzp, int passive_delay, - int polling_delay) +thermal_zone_device_register_with_trips(const char *type, + const struct thermal_trip *trips, + int num_trips, void *devdata, + const struct thermal_zone_device_ops *ops, + const struct thermal_zone_params *tzp, + int passive_delay, int polling_delay) { struct thermal_zone_device *tz; int id; @@ -1293,20 +1290,7 @@ thermal_zone_device_register_with_trips(const char *type, struct thermal_trip *t return ERR_PTR(-EINVAL); } - /* - * Max trip count can't exceed 31 as the "mask >> num_trips" condition. - * For example, shifting by 32 will result in compiler warning: - * warning: right shift count >= width of type [-Wshift-count- overflow] - * - * Also "mask >> num_trips" will always be true with 32 bit shift. - * E.g. mask = 0x80000000 for trip id 31 to be RW. Then - * mask >> 32 = 0x80000000 - * This will result in failure for the below condition. - * - * Check will be true when the bit 31 of the mask is set. - * 32 bit shift will cause overflow of 4 byte integer. - */ - if (num_trips > (BITS_PER_TYPE(int) - 1) || num_trips < 0 || mask >> num_trips) { + if (num_trips < 0) { pr_err("Incorrect number of thermal trips\n"); return ERR_PTR(-EINVAL); } @@ -1322,7 +1306,7 @@ thermal_zone_device_register_with_trips(const char *type, struct thermal_trip *t if (!thermal_class) return ERR_PTR(-ENODEV); - tz = kzalloc(sizeof(*tz), GFP_KERNEL); + tz = kzalloc(struct_size(tz, trips, num_trips), GFP_KERNEL); if (!tz) return ERR_PTR(-ENOMEM); @@ -1348,21 +1332,21 @@ thermal_zone_device_register_with_trips(const char *type, struct thermal_trip *t tz->id = id; strscpy(tz->type, type, sizeof(tz->type)); - if (!ops->critical) - ops->critical = thermal_zone_device_critical; + tz->ops = *ops; + if (!tz->ops.critical) + tz->ops.critical = thermal_zone_device_critical; - tz->ops = ops; tz->device.class = thermal_class; tz->devdata = devdata; - tz->trips = trips; tz->num_trips = num_trips; + memcpy(tz->trips, trips, num_trips * sizeof(*trips)); thermal_set_delay_jiffies(&tz->passive_delay_jiffies, passive_delay); thermal_set_delay_jiffies(&tz->polling_delay_jiffies, polling_delay); /* sys I/F */ /* Add nodes that are always present via .groups */ - result = thermal_zone_create_device_groups(tz, mask); + result = thermal_zone_create_device_groups(tz); if (result) goto remove_id; @@ -1437,10 +1421,10 @@ EXPORT_SYMBOL_GPL(thermal_zone_device_register_with_trips); struct thermal_zone_device *thermal_tripless_zone_device_register( const char *type, void *devdata, - struct thermal_zone_device_ops *ops, + const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp) { - return thermal_zone_device_register_with_trips(type, NULL, 0, 0, devdata, + return thermal_zone_device_register_with_trips(type, NULL, 0, devdata, ops, tzp, 0, 0); } EXPORT_SYMBOL_GPL(thermal_tripless_zone_device_register); @@ -1499,8 +1483,8 @@ void thermal_zone_device_unregister(struct thermal_zone_device *tz) /* Unbind all cdevs associated with 'this' thermal zone */ list_for_each_entry(cdev, &thermal_cdev_list, node) - if (tz->ops->unbind) - tz->ops->unbind(tz, cdev); + if (tz->ops.unbind) + tz->ops.unbind(tz, cdev); mutex_unlock(&thermal_list_lock); diff --git a/drivers/thermal/thermal_core.h b/drivers/thermal/thermal_core.h index e9c099ecdd0f..0d8a42bb7ce8 100644 --- a/drivers/thermal/thermal_core.h +++ b/drivers/thermal/thermal_core.h @@ -131,7 +131,7 @@ void thermal_zone_trip_updated(struct thermal_zone_device *tz, int __thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp); /* sysfs I/F */ -int thermal_zone_create_device_groups(struct thermal_zone_device *, int); +int thermal_zone_create_device_groups(struct thermal_zone_device *tz); void thermal_zone_destroy_device_groups(struct thermal_zone_device *); void thermal_cooling_device_setup_sysfs(struct thermal_cooling_device *); void thermal_cooling_device_destroy_sysfs(struct thermal_cooling_device *cdev); diff --git a/drivers/thermal/thermal_helpers.c b/drivers/thermal/thermal_helpers.c index 572c3fd73ee5..bc3d11fb3567 100644 --- a/drivers/thermal/thermal_helpers.c +++ b/drivers/thermal/thermal_helpers.c @@ -26,8 +26,8 @@ int get_tz_trend(struct thermal_zone_device *tz, const struct thermal_trip *trip { enum thermal_trend trend; - if (tz->emul_temperature || !tz->ops->get_trend || - tz->ops->get_trend(tz, trip, &trend)) { + if (tz->emul_temperature || !tz->ops.get_trend || + tz->ops.get_trend(tz, trip, &trend)) { if (tz->temperature > tz->last_temperature) trend = THERMAL_TREND_RAISING; else if (tz->temperature < tz->last_temperature) @@ -75,7 +75,7 @@ EXPORT_SYMBOL(get_thermal_instance); * temperature and fill @temp. * * Both tz and tz->ops must be valid pointers when calling this function, - * and the tz->ops->get_temp callback must be provided. + * and the tz->ops.get_temp callback must be provided. * The function must be called under tz->lock. * * Return: On success returns 0, an error code otherwise @@ -88,7 +88,7 @@ int __thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp) lockdep_assert_held(&tz->lock); - ret = tz->ops->get_temp(tz, temp); + ret = tz->ops.get_temp(tz, temp); if (IS_ENABLED(CONFIG_THERMAL_EMULATION) && tz->emul_temperature) { for_each_trip(tz, trip) { @@ -132,7 +132,7 @@ int thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp) mutex_lock(&tz->lock); - if (!tz->ops->get_temp) { + if (!tz->ops.get_temp) { ret = -EINVAL; goto unlock; } diff --git a/drivers/thermal/thermal_hwmon.c b/drivers/thermal/thermal_hwmon.c index 252116f1e535..f0e504fd866a 100644 --- a/drivers/thermal/thermal_hwmon.c +++ b/drivers/thermal/thermal_hwmon.c @@ -80,7 +80,7 @@ temp_crit_show(struct device *dev, struct device_attribute *attr, char *buf) mutex_lock(&tz->lock); - ret = tz->ops->get_crit_temp(tz, &temperature); + ret = tz->ops.get_crit_temp(tz, &temperature); mutex_unlock(&tz->lock); @@ -132,7 +132,7 @@ thermal_hwmon_lookup_temp(const struct thermal_hwmon_device *hwmon, static bool thermal_zone_crit_temp_valid(struct thermal_zone_device *tz) { int temp; - return tz->ops->get_crit_temp && !tz->ops->get_crit_temp(tz, &temp); + return tz->ops.get_crit_temp && !tz->ops.get_crit_temp(tz, &temp); } int thermal_add_hwmon_sysfs(struct thermal_zone_device *tz) diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c index 4d6c22e0ed85..f1cbf9aa62cf 100644 --- a/drivers/thermal/thermal_of.c +++ b/drivers/thermal/thermal_of.c @@ -117,6 +117,8 @@ static int thermal_of_populate_trip(struct device_node *np, return ret; } + trip->flags = THERMAL_TRIP_FLAG_RW_TEMP; + return 0; } @@ -438,13 +440,8 @@ static int thermal_of_unbind(struct thermal_zone_device *tz, */ static void thermal_of_zone_unregister(struct thermal_zone_device *tz) { - struct thermal_trip *trips = tz->trips; - struct thermal_zone_device_ops *ops = tz->ops; - thermal_zone_device_disable(tz); thermal_zone_device_unregister(tz); - kfree(trips); - kfree(ops); } /** @@ -470,33 +467,27 @@ static void thermal_of_zone_unregister(struct thermal_zone_device *tz) static struct thermal_zone_device *thermal_of_zone_register(struct device_node *sensor, int id, void *data, const struct thermal_zone_device_ops *ops) { + struct thermal_zone_device_ops of_ops = *ops; struct thermal_zone_device *tz; struct thermal_trip *trips; struct thermal_zone_params tzp = {}; - struct thermal_zone_device_ops *of_ops; struct device_node *np; const char *action; int delay, pdelay; - int ntrips, mask; + int ntrips; int ret; - of_ops = kmemdup(ops, sizeof(*ops), GFP_KERNEL); - if (!of_ops) - return ERR_PTR(-ENOMEM); - np = of_thermal_zone_find(sensor, id); if (IS_ERR(np)) { if (PTR_ERR(np) != -ENODEV) pr_err("Failed to find thermal zone for %pOFn id=%d\n", sensor, id); - ret = PTR_ERR(np); - goto out_kfree_of_ops; + return ERR_CAST(np); } trips = thermal_of_trips_init(np, &ntrips); if (IS_ERR(trips)) { pr_err("Failed to find trip points for %pOFn id=%d\n", sensor, id); - ret = PTR_ERR(trips); - goto out_kfree_of_ops; + return ERR_CAST(trips); } ret = thermal_of_monitor_init(np, &delay, &pdelay); @@ -507,18 +498,16 @@ static struct thermal_zone_device *thermal_of_zone_register(struct device_node * thermal_of_parameters_init(np, &tzp); - of_ops->bind = thermal_of_bind; - of_ops->unbind = thermal_of_unbind; - - mask = GENMASK_ULL((ntrips) - 1, 0); + of_ops.bind = thermal_of_bind; + of_ops.unbind = thermal_of_unbind; ret = of_property_read_string(np, "critical-action", &action); if (!ret) - if (!of_ops->critical && !strcasecmp(action, "reboot")) - of_ops->critical = thermal_zone_device_critical_reboot; + if (!of_ops.critical && !strcasecmp(action, "reboot")) + of_ops.critical = thermal_zone_device_critical_reboot; tz = thermal_zone_device_register_with_trips(np->name, trips, ntrips, - mask, data, of_ops, &tzp, + data, &of_ops, &tzp, pdelay, delay); if (IS_ERR(tz)) { ret = PTR_ERR(tz); @@ -526,6 +515,8 @@ static struct thermal_zone_device *thermal_of_zone_register(struct device_node * goto out_kfree_trips; } + kfree(trips); + ret = thermal_zone_device_enable(tz); if (ret) { pr_err("Failed to enabled thermal zone '%s', id=%d: %d\n", @@ -538,8 +529,6 @@ static struct thermal_zone_device *thermal_of_zone_register(struct device_node * out_kfree_trips: kfree(trips); -out_kfree_of_ops: - kfree(of_ops); return ERR_PTR(ret); } diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c index f4033865b093..5b533fa40437 100644 --- a/drivers/thermal/thermal_sysfs.c +++ b/drivers/thermal/thermal_sysfs.c @@ -123,8 +123,8 @@ trip_point_temp_store(struct device *dev, struct device_attribute *attr, trip = &tz->trips[trip_id]; if (temp != trip->temperature) { - if (tz->ops->set_trip_temp) { - ret = tz->ops->set_trip_temp(tz, trip_id, temp); + if (tz->ops.set_trip_temp) { + ret = tz->ops.set_trip_temp(tz, trip_id, temp); if (ret) goto unlock; } @@ -136,7 +136,7 @@ trip_point_temp_store(struct device *dev, struct device_attribute *attr, unlock: mutex_unlock(&tz->lock); - + return ret ? ret : count; } @@ -174,21 +174,14 @@ trip_point_hyst_store(struct device *dev, struct device_attribute *attr, trip = &tz->trips[trip_id]; if (hyst != trip->hysteresis) { - if (tz->ops->set_trip_hyst) { - ret = tz->ops->set_trip_hyst(tz, trip_id, hyst); - if (ret) - goto unlock; - } - trip->hysteresis = hyst; thermal_zone_trip_updated(tz, trip); } -unlock: mutex_unlock(&tz->lock); - return ret ? ret : count; + return count; } static ssize_t @@ -250,10 +243,10 @@ emul_temp_store(struct device *dev, struct device_attribute *attr, mutex_lock(&tz->lock); - if (!tz->ops->set_emul_temp) + if (!tz->ops.set_emul_temp) tz->emul_temperature = temperature; else - ret = tz->ops->set_emul_temp(tz, temperature); + ret = tz->ops.set_emul_temp(tz, temperature); if (!ret) __thermal_zone_device_update(tz, THERMAL_EVENT_UNSPECIFIED); @@ -392,17 +385,16 @@ static const struct attribute_group *thermal_zone_attribute_groups[] = { /** * create_trip_attrs() - create attributes for trip points * @tz: the thermal zone device - * @mask: Writeable trip point bitmap. * * helper function to instantiate sysfs entries for every trip * point and its properties of a struct thermal_zone_device. * * Return: 0 on success, the proper error value otherwise. */ -static int create_trip_attrs(struct thermal_zone_device *tz, int mask) +static int create_trip_attrs(struct thermal_zone_device *tz) { + const struct thermal_trip *trip; struct attribute **attrs; - int indx; /* This function works only for zones with at least one trip */ if (tz->num_trips <= 0) @@ -437,7 +429,9 @@ static int create_trip_attrs(struct thermal_zone_device *tz, int mask) return -ENOMEM; } - for (indx = 0; indx < tz->num_trips; indx++) { + for_each_trip(tz, trip) { + int indx = thermal_zone_trip_id(tz, trip); + /* create trip type attribute */ snprintf(tz->trip_type_attrs[indx].name, THERMAL_NAME_LENGTH, "trip_point_%d_type", indx); @@ -458,8 +452,7 @@ static int create_trip_attrs(struct thermal_zone_device *tz, int mask) tz->trip_temp_attrs[indx].name; tz->trip_temp_attrs[indx].attr.attr.mode = S_IRUGO; tz->trip_temp_attrs[indx].attr.show = trip_point_temp_show; - if (IS_ENABLED(CONFIG_THERMAL_WRITABLE_TRIPS) && - mask & (1 << indx)) { + if (trip->flags & THERMAL_TRIP_FLAG_RW_TEMP) { tz->trip_temp_attrs[indx].attr.attr.mode |= S_IWUSR; tz->trip_temp_attrs[indx].attr.store = trip_point_temp_store; @@ -474,7 +467,7 @@ static int create_trip_attrs(struct thermal_zone_device *tz, int mask) tz->trip_hyst_attrs[indx].name; tz->trip_hyst_attrs[indx].attr.attr.mode = S_IRUGO; tz->trip_hyst_attrs[indx].attr.show = trip_point_hyst_show; - if (tz->ops->set_trip_hyst) { + if (trip->flags & THERMAL_TRIP_FLAG_RW_HYST) { tz->trip_hyst_attrs[indx].attr.attr.mode |= S_IWUSR; tz->trip_hyst_attrs[indx].attr.store = trip_point_hyst_store; @@ -506,8 +499,7 @@ static void destroy_trip_attrs(struct thermal_zone_device *tz) kfree(tz->trips_attribute_group.attrs); } -int thermal_zone_create_device_groups(struct thermal_zone_device *tz, - int mask) +int thermal_zone_create_device_groups(struct thermal_zone_device *tz) { const struct attribute_group **groups; int i, size, result; @@ -523,7 +515,7 @@ int thermal_zone_create_device_groups(struct thermal_zone_device *tz, groups[i] = thermal_zone_attribute_groups[i]; if (tz->num_trips) { - result = create_trip_attrs(tz, mask); + result = create_trip_attrs(tz); if (result) { kfree(groups); diff --git a/drivers/thermal/thermal_trip.c b/drivers/thermal/thermal_trip.c index c875a26d5adf..09f6050dd041 100644 --- a/drivers/thermal/thermal_trip.c +++ b/drivers/thermal/thermal_trip.c @@ -70,7 +70,7 @@ void __thermal_zone_set_trips(struct thermal_zone_device *tz) lockdep_assert_held(&tz->lock); - if (!tz->ops->set_trips) + if (!tz->ops.set_trips) return; for_each_trip(tz, trip) { @@ -114,7 +114,7 @@ void __thermal_zone_set_trips(struct thermal_zone_device *tz) * Set a temperature window. When this window is left the driver * must inform the thermal core via thermal_zone_device_update. */ - ret = tz->ops->set_trips(tz, low, high); + ret = tz->ops.set_trips(tz, low, high); if (ret) dev_err(&tz->device, "Failed to set trips: %d\n", ret); } @@ -122,7 +122,7 @@ void __thermal_zone_set_trips(struct thermal_zone_device *tz) int __thermal_zone_get_trip(struct thermal_zone_device *tz, int trip_id, struct thermal_trip *trip) { - if (!tz || !tz->trips || trip_id < 0 || trip_id >= tz->num_trips || !trip) + if (!tz || trip_id < 0 || trip_id >= tz->num_trips || !trip) return -EINVAL; *trip = tz->trips[trip_id]; diff --git a/drivers/usb/gadget/udc/max3420_udc.c b/drivers/usb/gadget/udc/max3420_udc.c index 89e8cf2a2a7d..7349ea774adf 100644 --- a/drivers/usb/gadget/udc/max3420_udc.c +++ b/drivers/usb/gadget/udc/max3420_udc.c @@ -1201,7 +1201,7 @@ static int max3420_probe(struct spi_device *spi) int err, irq; u8 reg[8]; - if (spi->master->flags & SPI_CONTROLLER_HALF_DUPLEX) { + if (spi->controller->flags & SPI_CONTROLLER_HALF_DUPLEX) { dev_err(&spi->dev, "UDC needs full duplex to work\n"); return -EINVAL; } diff --git a/drivers/usb/isp1760/isp1760-hcd.c b/drivers/usb/isp1760/isp1760-hcd.c index 76862ba40f35..0e5e4cb74c87 100644 --- a/drivers/usb/isp1760/isp1760-hcd.c +++ b/drivers/usb/isp1760/isp1760-hcd.c @@ -2521,21 +2521,19 @@ static const struct hc_driver isp1760_hc_driver = { int __init isp1760_init_kmem_once(void) { urb_listitem_cachep = kmem_cache_create("isp1760_urb_listitem", - sizeof(struct urb_listitem), 0, SLAB_TEMPORARY | - SLAB_MEM_SPREAD, NULL); + sizeof(struct urb_listitem), 0, SLAB_TEMPORARY, NULL); if (!urb_listitem_cachep) return -ENOMEM; qtd_cachep = kmem_cache_create("isp1760_qtd", - sizeof(struct isp1760_qtd), 0, SLAB_TEMPORARY | - SLAB_MEM_SPREAD, NULL); + sizeof(struct isp1760_qtd), 0, SLAB_TEMPORARY, NULL); if (!qtd_cachep) goto destroy_urb_listitem; qh_cachep = kmem_cache_create("isp1760_qh", sizeof(struct isp1760_qh), - 0, SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL); + 0, SLAB_TEMPORARY, NULL); if (!qh_cachep) goto destroy_qtd; diff --git a/drivers/video/fbdev/mmp/hw/mmp_spi.c b/drivers/video/fbdev/mmp/hw/mmp_spi.c index 0f8f0312a7c4..cf23650d7f0b 100644 --- a/drivers/video/fbdev/mmp/hw/mmp_spi.c +++ b/drivers/video/fbdev/mmp/hw/mmp_spi.c @@ -32,7 +32,7 @@ static inline int lcd_spi_write(struct spi_device *spi, u32 data) int timeout = 100000, isr, ret = 0; u32 tmp; void __iomem *reg_base = (void __iomem *) - *(void **)spi_master_get_devdata(spi->master); + *(void **) spi_controller_get_devdata(spi->controller); /* clear ISR */ writel_relaxed(~SPI_IRQ_MASK, reg_base + SPU_IRQ_ISR); @@ -81,7 +81,7 @@ static inline int lcd_spi_write(struct spi_device *spi, u32 data) static int lcd_spi_setup(struct spi_device *spi) { void __iomem *reg_base = (void __iomem *) - *(void **)spi_master_get_devdata(spi->master); + *(void **) spi_controller_get_devdata(spi->controller); u32 tmp; tmp = CFG_SCLKCNT(16) | @@ -136,32 +136,32 @@ static int lcd_spi_one_transfer(struct spi_device *spi, struct spi_message *m) int lcd_spi_register(struct mmphw_ctrl *ctrl) { - struct spi_master *master; + struct spi_controller *ctlr; void **p_regbase; int err; - master = spi_alloc_master(ctrl->dev, sizeof(void *)); - if (!master) { + ctlr = spi_alloc_master(ctrl->dev, sizeof(void *)); + if (!ctlr) { dev_err(ctrl->dev, "unable to allocate SPI master\n"); return -ENOMEM; } - p_regbase = spi_master_get_devdata(master); + p_regbase = spi_controller_get_devdata(ctlr); *p_regbase = (void __force *)ctrl->reg_base; /* set bus num to 5 to avoid conflict with other spi hosts */ - master->bus_num = 5; - master->num_chipselect = 1; - master->setup = lcd_spi_setup; - master->transfer = lcd_spi_one_transfer; + ctlr->bus_num = 5; + ctlr->num_chipselect = 1; + ctlr->setup = lcd_spi_setup; + ctlr->transfer = lcd_spi_one_transfer; - err = spi_register_master(master); + err = spi_register_controller(ctlr); if (err < 0) { dev_err(ctrl->dev, "unable to register SPI master\n"); - spi_master_put(master); + spi_controller_put(ctlr); return err; } - dev_info(&master->dev, "registered\n"); + dev_info(&ctlr->dev, "registered\n"); return 0; } diff --git a/fs/9p/v9fs.c b/fs/9p/v9fs.c index 61dbe52bb3a3..281a1ed03a04 100644 --- a/fs/9p/v9fs.c +++ b/fs/9p/v9fs.c @@ -637,7 +637,7 @@ static int v9fs_init_inode_cache(void) v9fs_inode_cache = kmem_cache_create("v9fs_inode_cache", sizeof(struct v9fs_inode), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), v9fs_inode_init_once); if (!v9fs_inode_cache) return -ENOMEM; diff --git a/fs/adfs/super.c b/fs/adfs/super.c index e8bfc38239cd..9354b14bbfe3 100644 --- a/fs/adfs/super.c +++ b/fs/adfs/super.c @@ -249,7 +249,7 @@ static int __init init_inodecache(void) adfs_inode_cachep = kmem_cache_create("adfs_inode_cache", sizeof(struct adfs_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (adfs_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/attr.c b/fs/attr.c index 49d23b5dbab4..960a310581eb 100644 --- a/fs/attr.c +++ b/fs/attr.c @@ -16,8 +16,6 @@ #include #include #include -#include -#include #include "internal.h" @@ -502,8 +500,7 @@ int notify_change(struct mnt_idmap *idmap, struct dentry *dentry, if (!error) { fsnotify_change(dentry, ia_valid); - ima_inode_post_setattr(idmap, dentry); - evm_inode_post_setattr(dentry, ia_valid); + security_inode_post_setattr(idmap, dentry, ia_valid); } return error; diff --git a/fs/befs/linuxvfs.c b/fs/befs/linuxvfs.c index 2b4dda047450..d76f406d3b2e 100644 --- a/fs/befs/linuxvfs.c +++ b/fs/befs/linuxvfs.c @@ -435,8 +435,7 @@ befs_init_inodecache(void) { befs_inode_cachep = kmem_cache_create_usercopy("befs_inode_cache", sizeof(struct befs_inode_info), 0, - (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| - SLAB_ACCOUNT), + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, offsetof(struct befs_inode_info, i_data.symlink), sizeof_field(struct befs_inode_info, diff --git a/fs/bfs/inode.c b/fs/bfs/inode.c index 355957dbce39..db81570c9637 100644 --- a/fs/bfs/inode.c +++ b/fs/bfs/inode.c @@ -259,7 +259,7 @@ static int __init init_inodecache(void) bfs_inode_cachep = kmem_cache_create("bfs_inode_cache", sizeof(struct bfs_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (bfs_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/ceph/super.c b/fs/ceph/super.c index 5ec102f6b1ac..885cb5d4e771 100644 --- a/fs/ceph/super.c +++ b/fs/ceph/super.c @@ -928,36 +928,36 @@ static int __init init_caches(void) ceph_inode_cachep = kmem_cache_create("ceph_inode_info", sizeof(struct ceph_inode_info), __alignof__(struct ceph_inode_info), - SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| - SLAB_ACCOUNT, ceph_inode_init_once); + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + ceph_inode_init_once); if (!ceph_inode_cachep) return -ENOMEM; - ceph_cap_cachep = KMEM_CACHE(ceph_cap, SLAB_MEM_SPREAD); + ceph_cap_cachep = KMEM_CACHE(ceph_cap, 0); if (!ceph_cap_cachep) goto bad_cap; - ceph_cap_snap_cachep = KMEM_CACHE(ceph_cap_snap, SLAB_MEM_SPREAD); + ceph_cap_snap_cachep = KMEM_CACHE(ceph_cap_snap, 0); if (!ceph_cap_snap_cachep) goto bad_cap_snap; ceph_cap_flush_cachep = KMEM_CACHE(ceph_cap_flush, - SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD); + SLAB_RECLAIM_ACCOUNT); if (!ceph_cap_flush_cachep) goto bad_cap_flush; ceph_dentry_cachep = KMEM_CACHE(ceph_dentry_info, - SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD); + SLAB_RECLAIM_ACCOUNT); if (!ceph_dentry_cachep) goto bad_dentry; - ceph_file_cachep = KMEM_CACHE(ceph_file_info, SLAB_MEM_SPREAD); + ceph_file_cachep = KMEM_CACHE(ceph_file_info, 0); if (!ceph_file_cachep) goto bad_file; - ceph_dir_file_cachep = KMEM_CACHE(ceph_dir_file_info, SLAB_MEM_SPREAD); + ceph_dir_file_cachep = KMEM_CACHE(ceph_dir_file_info, 0); if (!ceph_dir_file_cachep) goto bad_dir_file; - ceph_mds_request_cachep = KMEM_CACHE(ceph_mds_request, SLAB_MEM_SPREAD); + ceph_mds_request_cachep = KMEM_CACHE(ceph_mds_request, 0); if (!ceph_mds_request_cachep) goto bad_mds_req; diff --git a/fs/coda/inode.c b/fs/coda/inode.c index a50356c541f6..6898dc621011 100644 --- a/fs/coda/inode.c +++ b/fs/coda/inode.c @@ -72,8 +72,8 @@ int __init coda_init_inodecache(void) { coda_inode_cachep = kmem_cache_create("coda_inode_cache", sizeof(struct coda_inode_info), 0, - SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| - SLAB_ACCOUNT, init_once); + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + init_once); if (coda_inode_cachep == NULL) return -ENOMEM; return 0; diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 6fbb1fba2d31..69308fd73e4a 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -857,7 +857,7 @@ static int __init erofs_module_init(void) erofs_inode_cachep = kmem_cache_create("erofs_inode", sizeof(struct erofs_inode), 0, - SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT, + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, erofs_inode_init_once); if (!erofs_inode_cachep) return -ENOMEM; diff --git a/fs/exfat/cache.c b/fs/exfat/cache.c index 5a2f119b7e8c..7cc200d89821 100644 --- a/fs/exfat/cache.c +++ b/fs/exfat/cache.c @@ -46,7 +46,7 @@ int exfat_cache_init(void) { exfat_cachep = kmem_cache_create("exfat_cache", sizeof(struct exfat_cache), - 0, SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD, + 0, SLAB_RECLAIM_ACCOUNT, exfat_cache_init_once); if (!exfat_cachep) return -ENOMEM; diff --git a/fs/exfat/super.c b/fs/exfat/super.c index fcb658267765..3d5ea2cfad66 100644 --- a/fs/exfat/super.c +++ b/fs/exfat/super.c @@ -813,7 +813,7 @@ static int __init init_exfat_fs(void) exfat_inode_cachep = kmem_cache_create("exfat_inode_cache", sizeof(struct exfat_inode_info), - 0, SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, + 0, SLAB_RECLAIM_ACCOUNT, exfat_inode_init_once); if (!exfat_inode_cachep) { err = -ENOMEM; diff --git a/fs/ext2/super.c b/fs/ext2/super.c index 01f9addc8b1f..cabea887314d 100644 --- a/fs/ext2/super.c +++ b/fs/ext2/super.c @@ -213,8 +213,7 @@ static int __init init_inodecache(void) { ext2_inode_cachep = kmem_cache_create_usercopy("ext2_inode_cache", sizeof(struct ext2_inode_info), 0, - (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| - SLAB_ACCOUNT), + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, offsetof(struct ext2_inode_info, i_data), sizeof_field(struct ext2_inode_info, i_data), init_once); diff --git a/fs/ext4/super.c b/fs/ext4/super.c index a8ba84eabab2..59c72b6dd153 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -1500,8 +1500,7 @@ static int __init init_inodecache(void) { ext4_inode_cachep = kmem_cache_create_usercopy("ext4_inode_cache", sizeof(struct ext4_inode_info), 0, - (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| - SLAB_ACCOUNT), + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, offsetof(struct ext4_inode_info, i_data), sizeof_field(struct ext4_inode_info, i_data), init_once); diff --git a/fs/fat/cache.c b/fs/fat/cache.c index 738e427e2d21..2af424e200b3 100644 --- a/fs/fat/cache.c +++ b/fs/fat/cache.c @@ -47,7 +47,7 @@ int __init fat_cache_init(void) { fat_cache_cachep = kmem_cache_create("fat_cache", sizeof(struct fat_cache), - 0, SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD, + 0, SLAB_RECLAIM_ACCOUNT, init_once); if (fat_cache_cachep == NULL) return -ENOMEM; diff --git a/fs/fat/inode.c b/fs/fat/inode.c index 5c813696d1ff..d9e6fbb6f246 100644 --- a/fs/fat/inode.c +++ b/fs/fat/inode.c @@ -787,7 +787,7 @@ static int __init fat_init_inodecache(void) fat_inode_cachep = kmem_cache_create("fat_inode_cache", sizeof(struct msdos_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (fat_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/file_table.c b/fs/file_table.c index 6925522faa0a..4f03beed4737 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -26,7 +26,6 @@ #include #include #include -#include #include #include @@ -414,7 +413,7 @@ static void __fput(struct file *file) eventpoll_release(file); locks_remove_file(file); - ima_file_free(file); + security_file_release(file); if (unlikely(file->f_flags & FASYNC)) { if (file->f_op->fasync) file->f_op->fasync(-1, file, 0); diff --git a/fs/freevxfs/vxfs_super.c b/fs/freevxfs/vxfs_super.c index e6e2a2185e7c..42e03b6b1cc7 100644 --- a/fs/freevxfs/vxfs_super.c +++ b/fs/freevxfs/vxfs_super.c @@ -307,7 +307,7 @@ vxfs_init(void) vxfs_inode_cachep = kmem_cache_create_usercopy("vxfs_inode", sizeof(struct vxfs_inode_info), 0, - SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD, + SLAB_RECLAIM_ACCOUNT, offsetof(struct vxfs_inode_info, vii_immed.vi_immed), sizeof_field(struct vxfs_inode_info, vii_immed.vi_immed), diff --git a/fs/gfs2/main.c b/fs/gfs2/main.c index 79be0cdc730c..04cadc02e5a6 100644 --- a/fs/gfs2/main.c +++ b/fs/gfs2/main.c @@ -111,7 +111,6 @@ static int __init init_gfs2_fs(void) gfs2_inode_cachep = kmem_cache_create("gfs2_inode", sizeof(struct gfs2_inode), 0, SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD| SLAB_ACCOUNT, gfs2_init_inode_once); if (!gfs2_inode_cachep) diff --git a/fs/hpfs/super.c b/fs/hpfs/super.c index 6b0ba3c1efba..314834a078e9 100644 --- a/fs/hpfs/super.c +++ b/fs/hpfs/super.c @@ -255,7 +255,7 @@ static int init_inodecache(void) hpfs_inode_cachep = kmem_cache_create("hpfs_inode_cache", sizeof(struct hpfs_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (hpfs_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/isofs/inode.c b/fs/isofs/inode.c index 3e4d53e26f94..25fca44149dd 100644 --- a/fs/isofs/inode.c +++ b/fs/isofs/inode.c @@ -93,7 +93,7 @@ static int __init init_inodecache(void) isofs_inode_cachep = kmem_cache_create("isofs_inode_cache", sizeof(struct iso_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (!isofs_inode_cachep) return -ENOMEM; diff --git a/fs/jffs2/super.c b/fs/jffs2/super.c index f99591a634b4..aede1be4dc0c 100644 --- a/fs/jffs2/super.c +++ b/fs/jffs2/super.c @@ -387,7 +387,7 @@ static int __init init_jffs2_fs(void) jffs2_inode_cachep = kmem_cache_create("jffs2_i", sizeof(struct jffs2_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), jffs2_i_init_once); if (!jffs2_inode_cachep) { pr_err("error: Failed to initialise inode cache\n"); diff --git a/fs/namei.c b/fs/namei.c index 97952440277b..ceb9ddf8dfdd 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include @@ -3642,7 +3641,7 @@ static int do_open(struct nameidata *nd, if (!error && !(file->f_mode & FMODE_OPENED)) error = vfs_open(&nd->path, file); if (!error) - error = ima_file_check(file, op->acc_mode); + error = security_file_post_open(file, op->acc_mode); if (!error && do_truncate) error = handle_truncate(idmap, file); if (unlikely(error > 0)) { @@ -3705,7 +3704,7 @@ static int vfs_tmpfile(struct mnt_idmap *idmap, inode->i_state |= I_LINKABLE; spin_unlock(&inode->i_lock); } - ima_post_create_tmpfile(idmap, inode); + security_inode_post_create_tmpfile(idmap, inode); return 0; } @@ -4051,8 +4050,6 @@ retry: case 0: case S_IFREG: error = vfs_create(idmap, path.dentry->d_inode, dentry, mode, true); - if (!error) - ima_post_path_mknod(idmap, dentry); break; case S_IFCHR: case S_IFBLK: error = vfs_mknod(idmap, path.dentry->d_inode, @@ -4063,6 +4060,11 @@ retry: dentry, mode, 0); break; } + + if (error) + goto out2; + + security_path_post_mknod(idmap, dentry); out2: done_path_create(&path, dentry); if (retry_estale(error, lookup_flags)) { diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index c03926a1cc73..7af5d270de28 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -1037,8 +1037,7 @@ int __init nfs_init_directcache(void) { nfs_direct_cachep = kmem_cache_create("nfs_direct_cache", sizeof(struct nfs_direct_req), - 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD), + 0, SLAB_RECLAIM_ACCOUNT, NULL); if (nfs_direct_cachep == NULL) return -ENOMEM; diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index ebb8d60e1152..93ea49a7eb61 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -2372,7 +2372,7 @@ static int __init nfs_init_inodecache(void) nfs_inode_cachep = kmem_cache_create("nfs_inode_cache", sizeof(struct nfs_inode), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (nfs_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/nfs/nfs42xattr.c b/fs/nfs/nfs42xattr.c index 49aaf28a6950..b6e3d8f77b91 100644 --- a/fs/nfs/nfs42xattr.c +++ b/fs/nfs/nfs42xattr.c @@ -1017,7 +1017,7 @@ int __init nfs4_xattr_cache_init(void) nfs4_xattr_cache_cachep = kmem_cache_create("nfs4_xattr_cache_cache", sizeof(struct nfs4_xattr_cache), 0, - (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD), + (SLAB_RECLAIM_ACCOUNT), nfs4_xattr_cache_init_once); if (nfs4_xattr_cache_cachep == NULL) return -ENOMEM; diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index 6a4c506038e0..6a9464262fae 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -895,7 +894,7 @@ __nfsd_open(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, goto out; } - host_err = ima_file_check(file, may_flags); + host_err = security_file_post_open(file, may_flags); if (host_err) { fput(file); goto out; diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c index cef5467fd928..9df7c20d066f 100644 --- a/fs/ntfs3/super.c +++ b/fs/ntfs3/super.c @@ -1825,7 +1825,7 @@ static int __init init_ntfs_fs(void) ntfs_inode_cachep = kmem_cache_create( "ntfs_inode_cache", sizeof(struct ntfs_inode), 0, - (SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT), + (SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT), init_once); if (!ntfs_inode_cachep) { err = -ENOMEM; diff --git a/fs/ocfs2/dlmfs/dlmfs.c b/fs/ocfs2/dlmfs/dlmfs.c index 85215162c9dd..7fc0e920eda7 100644 --- a/fs/ocfs2/dlmfs/dlmfs.c +++ b/fs/ocfs2/dlmfs/dlmfs.c @@ -578,7 +578,7 @@ static int __init init_dlmfs_fs(void) dlmfs_inode_cache = kmem_cache_create("dlmfs_inode_cache", sizeof(struct dlmfs_inode_private), 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), dlmfs_init_once); if (!dlmfs_inode_cache) { status = -ENOMEM; diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c index a70aff17d455..b3f860888e93 100644 --- a/fs/ocfs2/super.c +++ b/fs/ocfs2/super.c @@ -1706,18 +1706,17 @@ static int ocfs2_initialize_mem_caches(void) sizeof(struct ocfs2_inode_info), 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), ocfs2_inode_init_once); ocfs2_dquot_cachep = kmem_cache_create("ocfs2_dquot_cache", sizeof(struct ocfs2_dquot), 0, - (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD), + (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT), NULL); ocfs2_qf_chunk_cachep = kmem_cache_create("ocfs2_qf_chunk_cache", sizeof(struct ocfs2_quota_chunk), 0, - (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD), + (SLAB_RECLAIM_ACCOUNT), NULL); if (!ocfs2_inode_cachep || !ocfs2_dquot_cachep || !ocfs2_qf_chunk_cachep) { diff --git a/fs/open.c b/fs/open.c index 3d55fbdc2d7d..c2d152aef585 100644 --- a/fs/open.c +++ b/fs/open.c @@ -29,7 +29,6 @@ #include #include #include -#include #include #include #include diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index 36d4b8b1f784..a40fc7e05525 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -1503,7 +1503,7 @@ static int __init ovl_init(void) ovl_inode_cachep = kmem_cache_create("ovl_inode", sizeof(struct ovl_inode), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), ovl_inode_init_once); if (ovl_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/posix_acl.c b/fs/posix_acl.c index 6bf587d1a9b8..3f87297dbfdb 100644 --- a/fs/posix_acl.c +++ b/fs/posix_acl.c @@ -26,7 +26,6 @@ #include #include #include -#include #include #include @@ -1137,7 +1136,7 @@ retry_deleg: error = -EIO; if (!error) { fsnotify_xattr(dentry); - evm_inode_post_set_acl(dentry, acl_name, kacl); + security_inode_post_set_acl(dentry, acl_name, kacl); } out_inode_unlock: @@ -1245,7 +1244,7 @@ retry_deleg: error = -EIO; if (!error) { fsnotify_xattr(dentry); - evm_inode_post_remove_acl(idmap, dentry, acl_name); + security_inode_post_remove_acl(idmap, dentry, acl_name); } out_inode_unlock: diff --git a/fs/qnx4/inode.c b/fs/qnx4/inode.c index 7b5711f76709..d79841e94428 100644 --- a/fs/qnx4/inode.c +++ b/fs/qnx4/inode.c @@ -378,7 +378,7 @@ static int init_inodecache(void) qnx4_inode_cachep = kmem_cache_create("qnx4_inode_cache", sizeof(struct qnx4_inode_info), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (qnx4_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c index 1f0c754416b6..eb6e9d95dea1 100644 --- a/fs/quota/dquot.c +++ b/fs/quota/dquot.c @@ -2984,7 +2984,7 @@ static int __init dquot_init(void) dquot_cachep = kmem_cache_create("dquot", sizeof(struct dquot), sizeof(unsigned long) * 4, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_PANIC), + SLAB_PANIC), NULL); order = 0; diff --git a/fs/smb/client/cifsfs.c b/fs/smb/client/cifsfs.c index fb368b191eef..e0d8c79cdde1 100644 --- a/fs/smb/client/cifsfs.c +++ b/fs/smb/client/cifsfs.c @@ -1664,7 +1664,7 @@ cifs_init_inodecache(void) cifs_inode_cachep = kmem_cache_create("cifs_inode_cache", sizeof(struct cifsInodeInfo), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), cifs_init_once); if (cifs_inode_cachep == NULL) return -ENOMEM; diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c index d65ffad4c327..5545e6bf7d26 100644 --- a/fs/tracefs/inode.c +++ b/fs/tracefs/inode.c @@ -731,7 +731,6 @@ static int __init tracefs_init(void) tracefs_inode_cachep = kmem_cache_create("tracefs_inode_cache", sizeof(struct tracefs_inode), 0, (SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD| SLAB_ACCOUNT), init_once); if (!tracefs_inode_cachep) diff --git a/fs/ubifs/super.c b/fs/ubifs/super.c index d2881041b393..7f4031a15f4d 100644 --- a/fs/ubifs/super.c +++ b/fs/ubifs/super.c @@ -2434,8 +2434,8 @@ static int __init ubifs_init(void) ubifs_inode_slab = kmem_cache_create("ubifs_inode_slab", sizeof(struct ubifs_inode), 0, - SLAB_MEM_SPREAD | SLAB_RECLAIM_ACCOUNT | - SLAB_ACCOUNT, &inode_slab_ctor); + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + &inode_slab_ctor); if (!ubifs_inode_slab) return -ENOMEM; diff --git a/fs/udf/super.c b/fs/udf/super.c index 928a04d9d9e0..6f420f4ca005 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -177,7 +177,6 @@ static int __init init_inodecache(void) udf_inode_cachep = kmem_cache_create("udf_inode_cache", sizeof(struct udf_inode_info), 0, (SLAB_RECLAIM_ACCOUNT | - SLAB_MEM_SPREAD | SLAB_ACCOUNT), init_once); if (!udf_inode_cachep) diff --git a/fs/ufs/super.c b/fs/ufs/super.c index a480810cd4e3..44666afc6209 100644 --- a/fs/ufs/super.c +++ b/fs/ufs/super.c @@ -1470,8 +1470,7 @@ static int __init init_inodecache(void) { ufs_inode_cachep = kmem_cache_create_usercopy("ufs_inode_cache", sizeof(struct ufs_inode_info), 0, - (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| - SLAB_ACCOUNT), + (SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT), offsetof(struct ufs_inode_info, i_u1.i_symlink), sizeof_field(struct ufs_inode_info, i_u1.i_symlink), diff --git a/fs/vboxsf/super.c b/fs/vboxsf/super.c index 1fb8f4df60cb..cabe8ac4fefc 100644 --- a/fs/vboxsf/super.c +++ b/fs/vboxsf/super.c @@ -339,8 +339,7 @@ static int vboxsf_setup(void) vboxsf_inode_cachep = kmem_cache_create("vboxsf_inode_cache", sizeof(struct vboxsf_inode), 0, - (SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | - SLAB_ACCOUNT), + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, vboxsf_inode_init_once); if (!vboxsf_inode_cachep) { err = -ENOMEM; diff --git a/fs/xattr.c b/fs/xattr.c index 09d927603433..f8b643f91a98 100644 --- a/fs/xattr.c +++ b/fs/xattr.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -552,11 +551,11 @@ __vfs_removexattr_locked(struct mnt_idmap *idmap, goto out; error = __vfs_removexattr(idmap, dentry, name); + if (error) + return error; - if (!error) { - fsnotify_xattr(dentry); - evm_inode_post_removexattr(dentry, name); - } + fsnotify_xattr(dentry); + security_inode_post_removexattr(dentry, name); out: return error; diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 00fbd5b6e582..59c8c0541bdd 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -2043,8 +2043,7 @@ xfs_init_caches(void) xfs_buf_cache = kmem_cache_create("xfs_buf", sizeof(struct xfs_buf), 0, SLAB_HWCACHE_ALIGN | - SLAB_RECLAIM_ACCOUNT | - SLAB_MEM_SPREAD, + SLAB_RECLAIM_ACCOUNT, NULL); if (!xfs_buf_cache) goto out; @@ -2109,14 +2108,14 @@ xfs_init_caches(void) sizeof(struct xfs_inode), 0, (SLAB_HWCACHE_ALIGN | SLAB_RECLAIM_ACCOUNT | - SLAB_MEM_SPREAD | SLAB_ACCOUNT), + SLAB_ACCOUNT), xfs_fs_inode_init_once); if (!xfs_inode_cache) goto out_destroy_efi_cache; xfs_ili_cache = kmem_cache_create("xfs_ili", sizeof(struct xfs_inode_log_item), 0, - SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, + SLAB_RECLAIM_ACCOUNT, NULL); if (!xfs_ili_cache) goto out_destroy_inode_cache; diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index 236a6d88306f..c6a124e8d565 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -1422,7 +1422,7 @@ static int __init zonefs_init_inodecache(void) { zonefs_inode_cachep = kmem_cache_create("zonefs_inode_cache", sizeof(struct zonefs_inode_info), 0, - (SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT), + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, NULL); if (zonefs_inode_cachep == NULL) return -ENOMEM; diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h index e4d24d3f9abb..5de954e2b18a 100644 --- a/include/acpi/acpi_bus.h +++ b/include/acpi/acpi_bus.h @@ -582,7 +582,7 @@ void acpi_initialize_hp_context(struct acpi_device *adev, void (*uevent)(struct acpi_device *, u32)); /* acpi_device.dev.bus == &acpi_bus_type */ -extern struct bus_type acpi_bus_type; +extern const struct bus_type acpi_bus_type; int acpi_bus_for_each_dev(int (*fn)(struct device *, void *), void *data); int acpi_dev_for_each_child(struct acpi_device *adev, @@ -749,6 +749,7 @@ bool acpi_device_override_status(struct acpi_device *adev, unsigned long long *s bool acpi_quirk_skip_acpi_ac_and_battery(void); int acpi_install_cmos_rtc_space_handler(acpi_handle handle); void acpi_remove_cmos_rtc_space_handler(acpi_handle handle); +int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *skip); #else static inline bool acpi_device_override_status(struct acpi_device *adev, unsigned long long *status) @@ -766,23 +767,22 @@ static inline int acpi_install_cmos_rtc_space_handler(acpi_handle handle) static inline void acpi_remove_cmos_rtc_space_handler(acpi_handle handle) { } -#endif - -#if IS_ENABLED(CONFIG_X86_ANDROID_TABLETS) -bool acpi_quirk_skip_i2c_client_enumeration(struct acpi_device *adev); -int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *skip); -bool acpi_quirk_skip_gpio_event_handlers(void); -#else -static inline bool acpi_quirk_skip_i2c_client_enumeration(struct acpi_device *adev) -{ - return false; -} static inline int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *skip) { *skip = false; return 0; } +#endif + +#if IS_ENABLED(CONFIG_X86_ANDROID_TABLETS) +bool acpi_quirk_skip_i2c_client_enumeration(struct acpi_device *adev); +bool acpi_quirk_skip_gpio_event_handlers(void); +#else +static inline bool acpi_quirk_skip_i2c_client_enumeration(struct acpi_device *adev) +{ + return false; +} static inline bool acpi_quirk_skip_gpio_event_handlers(void) { return false; diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h index 3a0995f8bce8..930b6afba6f4 100644 --- a/include/acpi/cppc_acpi.h +++ b/include/acpi/cppc_acpi.h @@ -139,6 +139,7 @@ struct cppc_cpudata { #ifdef CONFIG_ACPI_CPPC_LIB extern int cppc_get_desired_perf(int cpunum, u64 *desired_perf); extern int cppc_get_nominal_perf(int cpunum, u64 *nominal_perf); +extern int cppc_get_highest_perf(int cpunum, u64 *highest_perf); extern int cppc_get_perf_ctrs(int cpu, struct cppc_perf_fb_ctrs *perf_fb_ctrs); extern int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls); extern int cppc_set_enable(int cpu, bool enable); @@ -167,6 +168,10 @@ static inline int cppc_get_nominal_perf(int cpunum, u64 *nominal_perf) { return -ENOTSUPP; } +static inline int cppc_get_highest_perf(int cpunum, u64 *highest_perf) +{ + return -ENOTSUPP; +} static inline int cppc_get_perf_ctrs(int cpu, struct cppc_perf_fb_ctrs *perf_fb_ctrs) { return -ENOTSUPP; diff --git a/include/dt-bindings/power/qcom-rpmpd.h b/include/dt-bindings/power/qcom-rpmpd.h index 7f4e2983a4c5..608087fb9a3d 100644 --- a/include/dt-bindings/power/qcom-rpmpd.h +++ b/include/dt-bindings/power/qcom-rpmpd.h @@ -308,6 +308,13 @@ #define MSM8953_VDDMX 5 #define MSM8953_VDDMX_AO 6 +/* MSM8974 Power Domain Indexes */ +#define MSM8974_VDDCX 0 +#define MSM8974_VDDCX_AO 1 +#define MSM8974_VDDCX_VFC 2 +#define MSM8974_VDDGFX 3 +#define MSM8974_VDDGFX_VFC 4 + /* MSM8976 Power Domain Indexes */ #define MSM8976_VDDCX 0 #define MSM8976_VDDCX_AO 1 diff --git a/include/linux/acpi.h b/include/linux/acpi.h index b7165e52b3c6..a170c389dd74 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1170,6 +1170,7 @@ static inline void acpi_ec_set_gpe_wake_mask(u8 action) {} #endif #ifdef CONFIG_ACPI +char *acpi_handle_path(acpi_handle handle); __printf(3, 4) void acpi_handle_printk(const char *level, acpi_handle handle, const char *fmt, ...); diff --git a/include/linux/amba/pl022.h b/include/linux/amba/pl022.h index 9bf58aac0df2..d7b07d0311e1 100644 --- a/include/linux/amba/pl022.h +++ b/include/linux/amba/pl022.h @@ -16,6 +16,7 @@ #ifndef _SSP_PL022_H #define _SSP_PL022_H +#include #include /** @@ -224,6 +225,7 @@ struct dma_chan; * struct pl022_ssp_master - device.platform_data for SPI controller devices. * @bus_id: identifier for this bus * @enable_dma: if true enables DMA driven transfers. + * @dma_filter: callback filter for dma_request_channel. * @dma_rx_param: parameter to locate an RX DMA channel. * @dma_tx_param: parameter to locate a TX DMA channel. * @autosuspend_delay: delay in ms following transfer completion before the @@ -235,7 +237,7 @@ struct dma_chan; struct pl022_ssp_controller { u16 bus_id; u8 enable_dma:1; - bool (*dma_filter)(struct dma_chan *chan, void *filter_param); + dma_filter_fn dma_filter; void *dma_rx_param; void *dma_tx_param; int autosuspend_delay; diff --git a/include/linux/amd-pstate.h b/include/linux/amd-pstate.h index 6ad02ad9c7b4..d21838835abd 100644 --- a/include/linux/amd-pstate.h +++ b/include/linux/amd-pstate.h @@ -39,11 +39,16 @@ struct amd_aperf_mperf { * @cppc_req_cached: cached performance request hints * @highest_perf: the maximum performance an individual processor may reach, * assuming ideal conditions + * For platforms that do not support the preferred core feature, the + * highest_pef may be configured with 166 or 255, to avoid max frequency + * calculated wrongly. we take the fixed value as the highest_perf. * @nominal_perf: the maximum sustained performance level of the processor, * assuming ideal operating conditions * @lowest_nonlinear_perf: the lowest performance level at which nonlinear power * savings are achieved * @lowest_perf: the absolute lowest performance level of the processor + * @prefcore_ranking: the preferred core ranking, the higher value indicates a higher + * priority. * @max_freq: the frequency that mapped to highest_perf * @min_freq: the frequency that mapped to lowest_perf * @nominal_freq: the frequency that mapped to nominal_perf @@ -52,6 +57,9 @@ struct amd_aperf_mperf { * @prev: Last Aperf/Mperf/tsc count value read from register * @freq: current cpu frequency value * @boost_supported: check whether the Processor or SBIOS supports boost mode + * @hw_prefcore: check whether HW supports preferred core featue. + * Only when hw_prefcore and early prefcore param are true, + * AMD P-State driver supports preferred core featue. * @epp_policy: Last saved policy used to set energy-performance preference * @epp_cached: Cached CPPC energy-performance preference value * @policy: Cpufreq policy value @@ -70,6 +78,7 @@ struct amd_cpudata { u32 nominal_perf; u32 lowest_nonlinear_perf; u32 lowest_perf; + u32 prefcore_ranking; u32 min_limit_perf; u32 max_limit_perf; u32 min_limit_freq; @@ -85,6 +94,7 @@ struct amd_cpudata { u64 freq; bool boost_supported; + bool hw_prefcore; /* EPP feature related attributes*/ s16 epp_policy; diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index afda5f24d3dd..836d57579828 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -263,6 +263,7 @@ static inline bool cpufreq_supports_freq_invariance(void) return false; } static inline void disable_cpufreq(void) { } +static inline void cpufreq_update_limits(unsigned int cpu) { } #endif #ifdef CONFIG_CPU_FREQ_STAT @@ -568,9 +569,7 @@ static inline unsigned long cpufreq_scale(unsigned long old, u_int div, /* * The polling frequency depends on the capability of the processor. Default - * polling frequency is 1000 times the transition latency of the processor. The - * ondemand governor will work on any processor with transition latency <= 10ms, - * using appropriate sampling rate. + * polling frequency is 1000 times the transition latency of the processor. */ #define LATENCY_MULTIPLIER (1000) @@ -694,26 +693,6 @@ struct cpufreq_frequency_table { * order */ }; -#if defined(CONFIG_CPU_FREQ) && defined(CONFIG_PM_OPP) -int dev_pm_opp_init_cpufreq_table(struct device *dev, - struct cpufreq_frequency_table **table); -void dev_pm_opp_free_cpufreq_table(struct device *dev, - struct cpufreq_frequency_table **table); -#else -static inline int dev_pm_opp_init_cpufreq_table(struct device *dev, - struct cpufreq_frequency_table - **table) -{ - return -EINVAL; -} - -static inline void dev_pm_opp_free_cpufreq_table(struct device *dev, - struct cpufreq_frequency_table - **table) -{ -} -#endif - /* * cpufreq_for_each_entry - iterate over a cpufreq_frequency_table * @pos: the cpufreq_frequency_table * to use as a loop cursor. @@ -1021,6 +1000,18 @@ static inline int cpufreq_table_find_index_c(struct cpufreq_policy *policy, efficiencies); } +static inline bool cpufreq_is_in_limits(struct cpufreq_policy *policy, int idx) +{ + unsigned int freq; + + if (idx < 0) + return false; + + freq = policy->freq_table[idx].frequency; + + return freq == clamp_val(freq, policy->min, policy->max); +} + static inline int cpufreq_frequency_table_target(struct cpufreq_policy *policy, unsigned int target_freq, unsigned int relation) @@ -1054,7 +1045,8 @@ retry: return 0; } - if (idx < 0 && efficiencies) { + /* Limit frequency index to honor policy->min/max */ + if (!cpufreq_is_in_limits(policy, idx) && efficiencies) { efficiencies = false; goto retry; } diff --git a/include/linux/dm-bufio.h b/include/linux/dm-bufio.h index 75e7d8cbb532..d1503b815a78 100644 --- a/include/linux/dm-bufio.h +++ b/include/linux/dm-bufio.h @@ -64,6 +64,9 @@ void dm_bufio_set_sector_offset(struct dm_bufio_client *c, sector_t start); void *dm_bufio_read(struct dm_bufio_client *c, sector_t block, struct dm_buffer **bp); +void *dm_bufio_read_with_ioprio(struct dm_bufio_client *c, sector_t block, + struct dm_buffer **bp, unsigned short ioprio); + /* * Like dm_bufio_read, but return buffer from cache, don't read * it. If the buffer is not in the cache, return NULL. @@ -86,6 +89,10 @@ void *dm_bufio_new(struct dm_bufio_client *c, sector_t block, void dm_bufio_prefetch(struct dm_bufio_client *c, sector_t block, unsigned int n_blocks); +void dm_bufio_prefetch_with_ioprio(struct dm_bufio_client *c, + sector_t block, unsigned int n_blocks, + unsigned short ioprio); + /* * Release a reference obtained with dm_bufio_{read,get,new}. The data * pointer and dm_buffer pointer is no longer valid after this call. diff --git a/include/linux/dm-io.h b/include/linux/dm-io.h index 7595142f3fc5..7b2968612b7e 100644 --- a/include/linux/dm-io.h +++ b/include/linux/dm-io.h @@ -80,7 +80,8 @@ void dm_io_client_destroy(struct dm_io_client *client); * error occurred doing io to the corresponding region. */ int dm_io(struct dm_io_request *io_req, unsigned int num_regions, - struct dm_io_region *region, unsigned int long *sync_error_bits); + struct dm_io_region *region, unsigned int long *sync_error_bits, + unsigned short ioprio); #endif /* __KERNEL__ */ #endif /* _LINUX_DM_IO_H */ diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index 88d91e087471..770755df852f 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -12,6 +13,7 @@ /** * struct em_perf_state - Performance state of a performance domain + * @performance: CPU performance (capacity) at a given frequency * @frequency: The frequency in KHz, for consistency with CPUFreq * @power: The power consumed at this level (by 1 CPU or by a registered * device). It can be a total power: static and dynamic. @@ -20,6 +22,7 @@ * @flags: see "em_perf_state flags" description below. */ struct em_perf_state { + unsigned long performance; unsigned long frequency; unsigned long power; unsigned long cost; @@ -36,9 +39,21 @@ struct em_perf_state { */ #define EM_PERF_STATE_INEFFICIENT BIT(0) +/** + * struct em_perf_table - Performance states table + * @rcu: RCU used for safe access and destruction + * @kref: Reference counter to track the users + * @state: List of performance states, in ascending order + */ +struct em_perf_table { + struct rcu_head rcu; + struct kref kref; + struct em_perf_state state[]; +}; + /** * struct em_perf_domain - Performance domain - * @table: List of performance states, in ascending order + * @em_table: Pointer to the runtime modifiable em_perf_table * @nr_perf_states: Number of performance states * @flags: See "em_perf_domain flags" * @cpus: Cpumask covering the CPUs of the domain. It's here @@ -53,7 +68,7 @@ struct em_perf_state { * field is unused. */ struct em_perf_domain { - struct em_perf_state *table; + struct em_perf_table __rcu *em_table; int nr_perf_states; unsigned long flags; unsigned long cpus[]; @@ -98,27 +113,6 @@ struct em_perf_domain { #define EM_MAX_NUM_CPUS 16 #endif -/* - * To avoid an overflow on 32bit machines while calculating the energy - * use a different order in the operation. First divide by the 'cpu_scale' - * which would reduce big value stored in the 'cost' field, then multiply by - * the 'sum_util'. This would allow to handle existing platforms, which have - * e.g. power ~1.3 Watt at max freq, so the 'cost' value > 1mln micro-Watts. - * In such scenario, where there are 4 CPUs in the Perf. Domain the 'sum_util' - * could be 4096, then multiplication: 'cost' * 'sum_util' would overflow. - * This reordering of operations has some limitations, we lose small - * precision in the estimation (comparing to 64bit platform w/o reordering). - * - * We are safe on 64bit machine. - */ -#ifdef CONFIG_64BIT -#define em_estimate_energy(cost, sum_util, scale_cpu) \ - (((cost) * (sum_util)) / (scale_cpu)) -#else -#define em_estimate_energy(cost, sum_util, scale_cpu) \ - (((cost) / (scale_cpu)) * (sum_util)) -#endif - struct em_data_callback { /** * active_power() - Provide power at the next performance state of @@ -168,40 +162,48 @@ struct em_data_callback { struct em_perf_domain *em_cpu_get(int cpu); struct em_perf_domain *em_pd_get(struct device *dev); +int em_dev_update_perf_domain(struct device *dev, + struct em_perf_table __rcu *new_table); int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states, struct em_data_callback *cb, cpumask_t *span, bool microwatts); void em_dev_unregister_perf_domain(struct device *dev); +struct em_perf_table __rcu *em_table_alloc(struct em_perf_domain *pd); +void em_table_free(struct em_perf_table __rcu *table); +int em_dev_compute_costs(struct device *dev, struct em_perf_state *table, + int nr_states); /** * em_pd_get_efficient_state() - Get an efficient performance state from the EM - * @pd : Performance domain for which we want an efficient frequency - * @freq : Frequency to map with the EM + * @table: List of performance states, in ascending order + * @nr_perf_states: Number of performance states + * @max_util: Max utilization to map with the EM + * @pd_flags: Performance Domain flags * * It is called from the scheduler code quite frequently and as a consequence * doesn't implement any check. * - * Return: An efficient performance state, high enough to meet @freq + * Return: An efficient performance state id, high enough to meet @max_util * requirement. */ -static inline -struct em_perf_state *em_pd_get_efficient_state(struct em_perf_domain *pd, - unsigned long freq) +static inline int +em_pd_get_efficient_state(struct em_perf_state *table, int nr_perf_states, + unsigned long max_util, unsigned long pd_flags) { struct em_perf_state *ps; int i; - for (i = 0; i < pd->nr_perf_states; i++) { - ps = &pd->table[i]; - if (ps->frequency >= freq) { - if (pd->flags & EM_PERF_DOMAIN_SKIP_INEFFICIENCIES && + for (i = 0; i < nr_perf_states; i++) { + ps = &table[i]; + if (ps->performance >= max_util) { + if (pd_flags & EM_PERF_DOMAIN_SKIP_INEFFICIENCIES && ps->flags & EM_PERF_STATE_INEFFICIENT) continue; - break; + return i; } } - return ps; + return nr_perf_states - 1; } /** @@ -224,9 +226,13 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, unsigned long max_util, unsigned long sum_util, unsigned long allowed_cpu_cap) { - unsigned long freq, ref_freq, scale_cpu; + struct em_perf_table *em_table; struct em_perf_state *ps; - int cpu; + int i; + +#ifdef CONFIG_SCHED_DEBUG + WARN_ONCE(!rcu_read_lock_held(), "EM: rcu read lock needed\n"); +#endif if (!sum_util) return 0; @@ -234,31 +240,30 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, /* * In order to predict the performance state, map the utilization of * the most utilized CPU of the performance domain to a requested - * frequency, like schedutil. Take also into account that the real - * frequency might be set lower (due to thermal capping). Thus, clamp + * performance, like schedutil. Take also into account that the real + * performance might be set lower (due to thermal capping). Thus, clamp * max utilization to the allowed CPU capacity before calculating - * effective frequency. + * effective performance. */ - cpu = cpumask_first(to_cpumask(pd->cpus)); - scale_cpu = arch_scale_cpu_capacity(cpu); - ref_freq = arch_scale_freq_ref(cpu); - + max_util = map_util_perf(max_util); max_util = min(max_util, allowed_cpu_cap); - freq = map_util_freq(max_util, ref_freq, scale_cpu); /* * Find the lowest performance state of the Energy Model above the - * requested frequency. + * requested performance. */ - ps = em_pd_get_efficient_state(pd, freq); + em_table = rcu_dereference(pd->em_table); + i = em_pd_get_efficient_state(em_table->state, pd->nr_perf_states, + max_util, pd->flags); + ps = &em_table->state[i]; /* - * The capacity of a CPU in the domain at the performance state (ps) - * can be computed as: + * The performance (capacity) of a CPU in the domain at the performance + * state (ps) can be computed as: * - * ps->freq * scale_cpu - * ps->cap = -------------------- (1) - * cpu_max_freq + * ps->freq * scale_cpu + * ps->performance = -------------------- (1) + * cpu_max_freq * * So, ignoring the costs of idle states (which are not available in * the EM), the energy consumed by this CPU at that performance state @@ -266,9 +271,10 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, * * ps->power * cpu_util * cpu_nrg = -------------------- (2) - * ps->cap + * ps->performance * - * since 'cpu_util / ps->cap' represents its percentage of busy time. + * since 'cpu_util / ps->performance' represents its percentage of busy + * time. * * NOTE: Although the result of this computation actually is in * units of power, it can be manipulated as an energy value @@ -278,9 +284,9 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, * By injecting (1) in (2), 'cpu_nrg' can be re-expressed as a product * of two terms: * - * ps->power * cpu_max_freq cpu_util - * cpu_nrg = ------------------------ * --------- (3) - * ps->freq scale_cpu + * ps->power * cpu_max_freq + * cpu_nrg = ------------------------ * cpu_util (3) + * ps->freq * scale_cpu * * The first term is static, and is stored in the em_perf_state struct * as 'ps->cost'. @@ -290,11 +296,9 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, * total energy of the domain (which is the simple sum of the energy of * all of its CPUs) can be factorized as: * - * ps->cost * \Sum cpu_util - * pd_nrg = ------------------------ (4) - * scale_cpu + * pd_nrg = ps->cost * \Sum cpu_util (4) */ - return em_estimate_energy(ps->cost, sum_util, scale_cpu); + return ps->cost * sum_util; } /** @@ -309,6 +313,23 @@ static inline int em_pd_nr_perf_states(struct em_perf_domain *pd) return pd->nr_perf_states; } +/** + * em_perf_state_from_pd() - Get the performance states table of perf. + * domain + * @pd : performance domain for which this must be done + * + * To use this function the rcu_read_lock() should be hold. After the usage + * of the performance states table is finished, the rcu_read_unlock() should + * be called. + * + * Return: the pointer to performance states table of the performance domain + */ +static inline +struct em_perf_state *em_perf_state_from_pd(struct em_perf_domain *pd) +{ + return rcu_dereference(pd->em_table)->state; +} + #else struct em_data_callback {}; #define EM_ADV_DATA_CB(_active_power_cb, _cost_cb) { } @@ -343,6 +364,29 @@ static inline int em_pd_nr_perf_states(struct em_perf_domain *pd) { return 0; } +static inline +struct em_perf_table __rcu *em_table_alloc(struct em_perf_domain *pd) +{ + return NULL; +} +static inline void em_table_free(struct em_perf_table __rcu *table) {} +static inline +int em_dev_update_perf_domain(struct device *dev, + struct em_perf_table __rcu *new_table) +{ + return -EINVAL; +} +static inline +struct em_perf_state *em_perf_state_from_pd(struct em_perf_domain *pd) +{ + return NULL; +} +static inline +int em_dev_compute_costs(struct device *dev, struct em_perf_state *table, + int nr_states) +{ + return -EINVAL; +} #endif #endif diff --git a/include/linux/evm.h b/include/linux/evm.h index 36ec884320d9..d48d6da32315 100644 --- a/include/linux/evm.h +++ b/include/linux/evm.h @@ -12,52 +12,12 @@ #include #include -struct integrity_iint_cache; - #ifdef CONFIG_EVM extern int evm_set_key(void *key, size_t keylen); extern enum integrity_status evm_verifyxattr(struct dentry *dentry, const char *xattr_name, void *xattr_value, - size_t xattr_value_len, - struct integrity_iint_cache *iint); -extern int evm_inode_setattr(struct mnt_idmap *idmap, - struct dentry *dentry, struct iattr *attr); -extern void evm_inode_post_setattr(struct dentry *dentry, int ia_valid); -extern int evm_inode_setxattr(struct mnt_idmap *idmap, - struct dentry *dentry, const char *name, - const void *value, size_t size); -extern void evm_inode_post_setxattr(struct dentry *dentry, - const char *xattr_name, - const void *xattr_value, - size_t xattr_value_len); -extern int evm_inode_copy_up_xattr(const char *name); -extern int evm_inode_removexattr(struct mnt_idmap *idmap, - struct dentry *dentry, const char *xattr_name); -extern void evm_inode_post_removexattr(struct dentry *dentry, - const char *xattr_name); -static inline void evm_inode_post_remove_acl(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *acl_name) -{ - evm_inode_post_removexattr(dentry, acl_name); -} -extern int evm_inode_set_acl(struct mnt_idmap *idmap, - struct dentry *dentry, const char *acl_name, - struct posix_acl *kacl); -static inline int evm_inode_remove_acl(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *acl_name) -{ - return evm_inode_set_acl(idmap, dentry, acl_name, NULL); -} -static inline void evm_inode_post_set_acl(struct dentry *dentry, - const char *acl_name, - struct posix_acl *kacl) -{ - return evm_inode_post_setxattr(dentry, acl_name, NULL, 0); -} - + size_t xattr_value_len); int evm_inode_init_security(struct inode *inode, struct inode *dir, const struct qstr *qstr, struct xattr *xattrs, int *xattr_count); @@ -85,85 +45,12 @@ static inline int evm_set_key(void *key, size_t keylen) static inline enum integrity_status evm_verifyxattr(struct dentry *dentry, const char *xattr_name, void *xattr_value, - size_t xattr_value_len, - struct integrity_iint_cache *iint) + size_t xattr_value_len) { return INTEGRITY_UNKNOWN; } #endif -static inline int evm_inode_setattr(struct mnt_idmap *idmap, - struct dentry *dentry, struct iattr *attr) -{ - return 0; -} - -static inline void evm_inode_post_setattr(struct dentry *dentry, int ia_valid) -{ - return; -} - -static inline int evm_inode_setxattr(struct mnt_idmap *idmap, - struct dentry *dentry, const char *name, - const void *value, size_t size) -{ - return 0; -} - -static inline void evm_inode_post_setxattr(struct dentry *dentry, - const char *xattr_name, - const void *xattr_value, - size_t xattr_value_len) -{ - return; -} - -static inline int evm_inode_copy_up_xattr(const char *name) -{ - return 0; -} - -static inline int evm_inode_removexattr(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *xattr_name) -{ - return 0; -} - -static inline void evm_inode_post_removexattr(struct dentry *dentry, - const char *xattr_name) -{ - return; -} - -static inline void evm_inode_post_remove_acl(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *acl_name) -{ - return; -} - -static inline int evm_inode_set_acl(struct mnt_idmap *idmap, - struct dentry *dentry, const char *acl_name, - struct posix_acl *kacl) -{ - return 0; -} - -static inline int evm_inode_remove_acl(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *acl_name) -{ - return 0; -} - -static inline void evm_inode_post_set_acl(struct dentry *dentry, - const char *acl_name, - struct posix_acl *kacl) -{ - return; -} - static inline int evm_inode_init_security(struct inode *inode, struct inode *dir, const struct qstr *qstr, struct xattr *xattrs, diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 7f75c9a51874..dc75f802e284 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -335,10 +335,12 @@ struct gpio_irq_chip { * (same as GPIO_LINE_DIRECTION_OUT / GPIO_LINE_DIRECTION_IN), * or negative error. It is recommended to always implement this * function, even on input-only or output-only gpio chips. - * @direction_input: configures signal "offset" as input, or returns error - * This can be omitted on input-only or output-only gpio chips. - * @direction_output: configures signal "offset" as output, or returns error - * This can be omitted on input-only or output-only gpio chips. + * @direction_input: configures signal "offset" as input, returns 0 on success + * or a negative error number. This can be omitted on input-only or + * output-only gpio chips. + * @direction_output: configures signal "offset" as output, returns 0 on + * success or a negative error number. This can be omitted on input-only + * or output-only gpio chips. * @get: returns value for signal "offset", 0=low, 1=high, or negative error * @get_multiple: reads values for multiple signals defined by "mask" and * stores them in "bits", returns 0 on success or negative error @@ -549,6 +551,21 @@ DEFINE_CLASS(_gpiochip_for_each_data, }), const char **label, int *i) +/** + * for_each_hwgpio - Iterates over all GPIOs for given chip. + * @_chip: Chip to iterate over. + * @_i: Loop counter. + * @_label: Place to store the address of the label if the GPIO is requested. + * Set to NULL for unused GPIOs. + */ +#define for_each_hwgpio(_chip, _i, _label) \ + for (CLASS(_gpiochip_for_each_data, _data)(&_label, &_i); \ + *_data.i < _chip->ngpio; \ + (*_data.i)++, kfree(*(_data.label)), *_data.label = NULL) \ + if (IS_ERR(*_data.label = \ + gpiochip_dup_line_label(_chip, *_data.i))) {} \ + else + /** * for_each_requested_gpio_in_range - iterates over requested GPIOs in a given range * @_chip: the chip to query @@ -626,8 +643,9 @@ int devm_gpiochip_add_data_with_key(struct device *dev, struct gpio_chip *gc, void *data, struct lock_class_key *lock_key, struct lock_class_key *request_key); -struct gpio_device *gpio_device_find(void *data, - int (*match)(struct gpio_chip *gc, void *data)); +struct gpio_device *gpio_device_find(const void *data, + int (*match)(struct gpio_chip *gc, + const void *data)); struct gpio_device *gpio_device_find_by_label(const char *label); struct gpio_device *gpio_device_find_by_fwnode(const struct fwnode_handle *fwnode); @@ -704,18 +722,6 @@ int bgpio_init(struct gpio_chip *gc, struct device *dev, #define BGPIOF_NO_OUTPUT BIT(5) /* only input */ #define BGPIOF_NO_SET_ON_INPUT BIT(6) -int gpiochip_irq_map(struct irq_domain *d, unsigned int irq, - irq_hw_number_t hwirq); -void gpiochip_irq_unmap(struct irq_domain *d, unsigned int irq); - -int gpiochip_irq_domain_activate(struct irq_domain *domain, - struct irq_data *data, bool reserve); -void gpiochip_irq_domain_deactivate(struct irq_domain *domain, - struct irq_data *data); - -bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gc, - unsigned int offset); - #ifdef CONFIG_GPIOLIB_IRQCHIP int gpiochip_irqchip_add_domain(struct gpio_chip *gc, struct irq_domain *domain); diff --git a/include/linux/hwmon.h b/include/linux/hwmon.h index 8cd6a6b33593..edf96f249eb5 100644 --- a/include/linux/hwmon.h +++ b/include/linux/hwmon.h @@ -141,6 +141,7 @@ enum hwmon_in_attributes { hwmon_in_rated_min, hwmon_in_rated_max, hwmon_in_beep, + hwmon_in_fault, }; #define HWMON_I_ENABLE BIT(hwmon_in_enable) @@ -162,6 +163,7 @@ enum hwmon_in_attributes { #define HWMON_I_RATED_MIN BIT(hwmon_in_rated_min) #define HWMON_I_RATED_MAX BIT(hwmon_in_rated_max) #define HWMON_I_BEEP BIT(hwmon_in_beep) +#define HWMON_I_FAULT BIT(hwmon_in_fault) enum hwmon_curr_attributes { hwmon_curr_enable, @@ -293,6 +295,8 @@ enum hwmon_humidity_attributes { hwmon_humidity_fault, hwmon_humidity_rated_min, hwmon_humidity_rated_max, + hwmon_humidity_min_alarm, + hwmon_humidity_max_alarm, }; #define HWMON_H_ENABLE BIT(hwmon_humidity_enable) @@ -306,6 +310,8 @@ enum hwmon_humidity_attributes { #define HWMON_H_FAULT BIT(hwmon_humidity_fault) #define HWMON_H_RATED_MIN BIT(hwmon_humidity_rated_min) #define HWMON_H_RATED_MAX BIT(hwmon_humidity_rated_max) +#define HWMON_H_MIN_ALARM BIT(hwmon_humidity_min_alarm) +#define HWMON_H_MAX_ALARM BIT(hwmon_humidity_max_alarm) enum hwmon_fan_attributes { hwmon_fan_enable, @@ -425,12 +431,12 @@ struct hwmon_channel_info { const u32 *config; }; -#define HWMON_CHANNEL_INFO(stype, ...) \ - (&(struct hwmon_channel_info) { \ - .type = hwmon_##stype, \ - .config = (u32 []) { \ - __VA_ARGS__, 0 \ - } \ +#define HWMON_CHANNEL_INFO(stype, ...) \ + (&(const struct hwmon_channel_info) { \ + .type = hwmon_##stype, \ + .config = (const u32 []) { \ + __VA_ARGS__, 0 \ + } \ }) /** diff --git a/include/linux/ima.h b/include/linux/ima.h index 86b57757c7b1..0bae61a15b60 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -16,23 +16,6 @@ struct linux_binprm; #ifdef CONFIG_IMA extern enum hash_algo ima_get_current_hash_algo(void); -extern int ima_bprm_check(struct linux_binprm *bprm); -extern int ima_file_check(struct file *file, int mask); -extern void ima_post_create_tmpfile(struct mnt_idmap *idmap, - struct inode *inode); -extern void ima_file_free(struct file *file); -extern int ima_file_mmap(struct file *file, unsigned long reqprot, - unsigned long prot, unsigned long flags); -extern int ima_file_mprotect(struct vm_area_struct *vma, unsigned long prot); -extern int ima_load_data(enum kernel_load_data_id id, bool contents); -extern int ima_post_load_data(char *buf, loff_t size, - enum kernel_load_data_id id, char *description); -extern int ima_read_file(struct file *file, enum kernel_read_file_id id, - bool contents); -extern int ima_post_read_file(struct file *file, void *buf, loff_t size, - enum kernel_read_file_id id); -extern void ima_post_path_mknod(struct mnt_idmap *idmap, - struct dentry *dentry); extern int ima_file_hash(struct file *file, char *buf, size_t buf_size); extern int ima_inode_hash(struct inode *inode, char *buf, size_t buf_size); extern void ima_kexec_cmdline(int kernel_fd, const void *buf, int size); @@ -57,68 +40,6 @@ static inline enum hash_algo ima_get_current_hash_algo(void) return HASH_ALGO__LAST; } -static inline int ima_bprm_check(struct linux_binprm *bprm) -{ - return 0; -} - -static inline int ima_file_check(struct file *file, int mask) -{ - return 0; -} - -static inline void ima_post_create_tmpfile(struct mnt_idmap *idmap, - struct inode *inode) -{ -} - -static inline void ima_file_free(struct file *file) -{ - return; -} - -static inline int ima_file_mmap(struct file *file, unsigned long reqprot, - unsigned long prot, unsigned long flags) -{ - return 0; -} - -static inline int ima_file_mprotect(struct vm_area_struct *vma, - unsigned long prot) -{ - return 0; -} - -static inline int ima_load_data(enum kernel_load_data_id id, bool contents) -{ - return 0; -} - -static inline int ima_post_load_data(char *buf, loff_t size, - enum kernel_load_data_id id, - char *description) -{ - return 0; -} - -static inline int ima_read_file(struct file *file, enum kernel_read_file_id id, - bool contents) -{ - return 0; -} - -static inline int ima_post_read_file(struct file *file, void *buf, loff_t size, - enum kernel_read_file_id id) -{ - return 0; -} - -static inline void ima_post_path_mknod(struct mnt_idmap *idmap, - struct dentry *dentry) -{ - return; -} - static inline int ima_file_hash(struct file *file, char *buf, size_t buf_size) { return -EOPNOTSUPP; @@ -169,76 +90,13 @@ static inline void ima_add_kexec_buffer(struct kimage *image) {} #endif -#ifdef CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS -extern void ima_post_key_create_or_update(struct key *keyring, - struct key *key, - const void *payload, size_t plen, - unsigned long flags, bool create); -#else -static inline void ima_post_key_create_or_update(struct key *keyring, - struct key *key, - const void *payload, - size_t plen, - unsigned long flags, - bool create) {} -#endif /* CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS */ - #ifdef CONFIG_IMA_APPRAISE extern bool is_ima_appraise_enabled(void); -extern void ima_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry); -extern int ima_inode_setxattr(struct dentry *dentry, const char *xattr_name, - const void *xattr_value, size_t xattr_value_len); -extern int ima_inode_set_acl(struct mnt_idmap *idmap, - struct dentry *dentry, const char *acl_name, - struct posix_acl *kacl); -static inline int ima_inode_remove_acl(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *acl_name) -{ - return ima_inode_set_acl(idmap, dentry, acl_name, NULL); -} -extern int ima_inode_removexattr(struct dentry *dentry, const char *xattr_name); #else static inline bool is_ima_appraise_enabled(void) { return 0; } - -static inline void ima_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry) -{ - return; -} - -static inline int ima_inode_setxattr(struct dentry *dentry, - const char *xattr_name, - const void *xattr_value, - size_t xattr_value_len) -{ - return 0; -} - -static inline int ima_inode_set_acl(struct mnt_idmap *idmap, - struct dentry *dentry, const char *acl_name, - struct posix_acl *kacl) -{ - - return 0; -} - -static inline int ima_inode_removexattr(struct dentry *dentry, - const char *xattr_name) -{ - return 0; -} - -static inline int ima_inode_remove_acl(struct mnt_idmap *idmap, - struct dentry *dentry, - const char *acl_name) -{ - return 0; -} #endif /* CONFIG_IMA_APPRAISE */ #if defined(CONFIG_IMA_APPRAISE) && defined(CONFIG_INTEGRITY_TRUSTED_KEYRING) diff --git a/include/linux/integrity.h b/include/linux/integrity.h index 2ea0f2f65ab6..459b79683783 100644 --- a/include/linux/integrity.h +++ b/include/linux/integrity.h @@ -19,40 +19,13 @@ enum integrity_status { INTEGRITY_UNKNOWN, }; -/* List of EVM protected security xattrs */ #ifdef CONFIG_INTEGRITY -extern struct integrity_iint_cache *integrity_inode_get(struct inode *inode); -extern void integrity_inode_free(struct inode *inode); extern void __init integrity_load_keys(void); #else -static inline struct integrity_iint_cache * - integrity_inode_get(struct inode *inode) -{ - return NULL; -} - -static inline void integrity_inode_free(struct inode *inode) -{ - return; -} - static inline void integrity_load_keys(void) { } #endif /* CONFIG_INTEGRITY */ -#ifdef CONFIG_INTEGRITY_ASYMMETRIC_KEYS - -extern int integrity_kernel_module_request(char *kmod_name); - -#else - -static inline int integrity_kernel_module_request(char *kmod_name) -{ - return 0; -} - -#endif /* CONFIG_INTEGRITY_ASYMMETRIC_KEYS */ - #endif /* _LINUX_INTEGRITY_H */ diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index 33f21bd85dbf..f3196f82fd8a 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -178,6 +178,12 @@ struct rapl_package { struct rapl_if_priv *priv; }; +struct rapl_package *rapl_find_package_domain_cpuslocked(int id, struct rapl_if_priv *priv, + bool id_is_cpu); +struct rapl_package *rapl_add_package_cpuslocked(int id, struct rapl_if_priv *priv, + bool id_is_cpu); +void rapl_remove_package_cpuslocked(struct rapl_package *rp); + struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu); struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu); void rapl_remove_package(struct rapl_package *rp); diff --git a/include/linux/intel_tcc.h b/include/linux/intel_tcc.h index f422612c28d6..8ff8eabb4a98 100644 --- a/include/linux/intel_tcc.h +++ b/include/linux/intel_tcc.h @@ -13,6 +13,6 @@ int intel_tcc_get_tjmax(int cpu); int intel_tcc_get_offset(int cpu); int intel_tcc_set_offset(int cpu, int offset); -int intel_tcc_get_temp(int cpu, bool pkg); +int intel_tcc_get_temp(int cpu, int *temp, bool pkg); #endif /* __INTEL_TCC_H__ */ diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 5e27cb3a3be9..2e925b5eba53 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -14,7 +14,6 @@ #include #include #include -#include #define IOMMU_READ (1 << 0) #define IOMMU_WRITE (1 << 1) @@ -41,8 +40,110 @@ struct iommu_domain_ops; struct iommu_dirty_ops; struct notifier_block; struct iommu_sva; -struct iommu_fault_event; struct iommu_dma_cookie; +struct iommu_fault_param; + +#define IOMMU_FAULT_PERM_READ (1 << 0) /* read */ +#define IOMMU_FAULT_PERM_WRITE (1 << 1) /* write */ +#define IOMMU_FAULT_PERM_EXEC (1 << 2) /* exec */ +#define IOMMU_FAULT_PERM_PRIV (1 << 3) /* privileged */ + +/* Generic fault types, can be expanded IRQ remapping fault */ +enum iommu_fault_type { + IOMMU_FAULT_PAGE_REQ = 1, /* page request fault */ +}; + +/** + * struct iommu_fault_page_request - Page Request data + * @flags: encodes whether the corresponding fields are valid and whether this + * is the last page in group (IOMMU_FAULT_PAGE_REQUEST_* values). + * When IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID is set, the page response + * must have the same PASID value as the page request. When it is clear, + * the page response should not have a PASID. + * @pasid: Process Address Space ID + * @grpid: Page Request Group Index + * @perm: requested page permissions (IOMMU_FAULT_PERM_* values) + * @addr: page address + * @private_data: device-specific private information + */ +struct iommu_fault_page_request { +#define IOMMU_FAULT_PAGE_REQUEST_PASID_VALID (1 << 0) +#define IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE (1 << 1) +#define IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA (1 << 2) +#define IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID (1 << 3) + u32 flags; + u32 pasid; + u32 grpid; + u32 perm; + u64 addr; + u64 private_data[2]; +}; + +/** + * struct iommu_fault - Generic fault data + * @type: fault type from &enum iommu_fault_type + * @prm: Page Request message, when @type is %IOMMU_FAULT_PAGE_REQ + */ +struct iommu_fault { + u32 type; + struct iommu_fault_page_request prm; +}; + +/** + * enum iommu_page_response_code - Return status of fault handlers + * @IOMMU_PAGE_RESP_SUCCESS: Fault has been handled and the page tables + * populated, retry the access. This is "Success" in PCI PRI. + * @IOMMU_PAGE_RESP_FAILURE: General error. Drop all subsequent faults from + * this device if possible. This is "Response Failure" in PCI PRI. + * @IOMMU_PAGE_RESP_INVALID: Could not handle this fault, don't retry the + * access. This is "Invalid Request" in PCI PRI. + */ +enum iommu_page_response_code { + IOMMU_PAGE_RESP_SUCCESS = 0, + IOMMU_PAGE_RESP_INVALID, + IOMMU_PAGE_RESP_FAILURE, +}; + +/** + * struct iommu_page_response - Generic page response information + * @pasid: Process Address Space ID + * @grpid: Page Request Group Index + * @code: response code from &enum iommu_page_response_code + */ +struct iommu_page_response { + u32 pasid; + u32 grpid; + u32 code; +}; + +struct iopf_fault { + struct iommu_fault fault; + /* node for pending lists */ + struct list_head list; +}; + +struct iopf_group { + struct iopf_fault last_fault; + struct list_head faults; + /* list node for iommu_fault_param::faults */ + struct list_head pending_node; + struct work_struct work; + struct iommu_domain *domain; + /* The device's fault data parameter. */ + struct iommu_fault_param *fault_param; +}; + +/** + * struct iopf_queue - IO Page Fault queue + * @wq: the fault workqueue + * @devices: devices attached to this queue + * @lock: protects the device list + */ +struct iopf_queue { + struct workqueue_struct *wq; + struct list_head devices; + struct mutex lock; +}; /* iommu fault flags */ #define IOMMU_FAULT_READ 0x0 @@ -50,7 +151,6 @@ struct iommu_dma_cookie; typedef int (*iommu_fault_handler_t)(struct iommu_domain *, struct device *, unsigned long, int, void *); -typedef int (*iommu_dev_fault_handler_t)(struct iommu_fault *, void *); struct iommu_domain_geometry { dma_addr_t aperture_start; /* First address that can be mapped */ @@ -110,8 +210,7 @@ struct iommu_domain { unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */ struct iommu_domain_geometry geometry; struct iommu_dma_cookie *iova_cookie; - enum iommu_page_response_code (*iopf_handler)(struct iommu_fault *fault, - void *data); + int (*iopf_handler)(struct iopf_group *group); void *fault_data; union { struct { @@ -468,16 +567,15 @@ struct iommu_ops { /* Request/Free a list of reserved regions for a device */ void (*get_resv_regions)(struct device *dev, struct list_head *list); - int (*of_xlate)(struct device *dev, struct of_phandle_args *args); + int (*of_xlate)(struct device *dev, const struct of_phandle_args *args); bool (*is_attach_deferred)(struct device *dev); /* Per device IOMMU features */ int (*dev_enable_feat)(struct device *dev, enum iommu_dev_features f); int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f); - int (*page_response)(struct device *dev, - struct iommu_fault_event *evt, - struct iommu_page_response *msg); + void (*page_response)(struct device *dev, struct iopf_fault *evt, + struct iommu_page_response *msg); int (*def_domain_type)(struct device *dev); void (*remove_dev_pasid)(struct device *dev, ioasid_t pasid); @@ -487,6 +585,7 @@ struct iommu_ops { struct module *owner; struct iommu_domain *identity_domain; struct iommu_domain *blocked_domain; + struct iommu_domain *release_domain; struct iommu_domain *default_domain; }; @@ -577,39 +676,35 @@ struct iommu_device { u32 max_pasids; }; -/** - * struct iommu_fault_event - Generic fault event - * - * Can represent recoverable faults such as a page requests or - * unrecoverable faults such as DMA or IRQ remapping faults. - * - * @fault: fault descriptor - * @list: pending fault event list, used for tracking responses - */ -struct iommu_fault_event { - struct iommu_fault fault; - struct list_head list; -}; - /** * struct iommu_fault_param - per-device IOMMU fault data - * @handler: Callback function to handle IOMMU faults at device level - * @data: handler private data - * @faults: holds the pending faults which needs response * @lock: protect pending faults list + * @users: user counter to manage the lifetime of the data + * @rcu: rcu head for kfree_rcu() + * @dev: the device that owns this param + * @queue: IOPF queue + * @queue_list: index into queue->devices + * @partial: faults that are part of a Page Request Group for which the last + * request hasn't been submitted yet. + * @faults: holds the pending faults which need response */ struct iommu_fault_param { - iommu_dev_fault_handler_t handler; - void *data; - struct list_head faults; struct mutex lock; + refcount_t users; + struct rcu_head rcu; + + struct device *dev; + struct iopf_queue *queue; + struct list_head queue_list; + + struct list_head partial; + struct list_head faults; }; /** * struct dev_iommu - Collection of per-device IOMMU data * * @fault_param: IOMMU detected device fault reporting data - * @iopf_param: I/O Page Fault queue and data * @fwspec: IOMMU fwspec data * @iommu_dev: IOMMU device this device is linked to * @priv: IOMMU Driver private data @@ -624,8 +719,7 @@ struct iommu_fault_param { */ struct dev_iommu { struct mutex lock; - struct iommu_fault_param *fault_param; - struct iopf_device_param *iopf_param; + struct iommu_fault_param __rcu *fault_param; struct iommu_fwspec *fwspec; struct iommu_device *iommu_dev; void *priv; @@ -654,6 +748,22 @@ static inline struct iommu_device *dev_to_iommu_device(struct device *dev) return (struct iommu_device *)dev_get_drvdata(dev); } +/** + * iommu_get_iommu_dev - Get iommu_device for a device + * @dev: an end-point device + * + * Note that this function must be called from the iommu_ops + * to retrieve the iommu_device for a device, which the core code + * guarentees it will not invoke the op without an attached iommu. + */ +static inline struct iommu_device *__iommu_get_iommu_dev(struct device *dev) +{ + return dev->iommu->iommu_dev; +} + +#define iommu_get_iommu_dev(dev, type, member) \ + container_of(__iommu_get_iommu_dev(dev), type, member) + static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather) { *gather = (struct iommu_iotlb_gather) { @@ -719,16 +829,6 @@ extern int iommu_group_for_each_dev(struct iommu_group *group, void *data, extern struct iommu_group *iommu_group_get(struct device *dev); extern struct iommu_group *iommu_group_ref_get(struct iommu_group *group); extern void iommu_group_put(struct iommu_group *group); -extern int iommu_register_device_fault_handler(struct device *dev, - iommu_dev_fault_handler_t handler, - void *data); - -extern int iommu_unregister_device_fault_handler(struct device *dev); - -extern int iommu_report_device_fault(struct device *dev, - struct iommu_fault_event *evt); -extern int iommu_page_response(struct device *dev, - struct iommu_page_response *msg); extern int iommu_group_id(struct iommu_group *group); extern struct iommu_domain *iommu_group_default_domain(struct iommu_group *); @@ -905,8 +1005,8 @@ struct iommu_mm_data { int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, const struct iommu_ops *ops); void iommu_fwspec_free(struct device *dev); -int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids); -const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode); +int iommu_fwspec_add_ids(struct device *dev, const u32 *ids, int num_ids); +const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode); static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) { @@ -948,8 +1048,6 @@ bool iommu_group_dma_owner_claimed(struct iommu_group *group); int iommu_device_claim_dma_owner(struct device *dev, void *owner); void iommu_device_release_dma_owner(struct device *dev); -struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, - struct mm_struct *mm); int iommu_attach_device_pasid(struct iommu_domain *domain, struct device *dev, ioasid_t pasid); void iommu_detach_device_pasid(struct iommu_domain *domain, @@ -1138,31 +1236,6 @@ static inline void iommu_group_put(struct iommu_group *group) { } -static inline -int iommu_register_device_fault_handler(struct device *dev, - iommu_dev_fault_handler_t handler, - void *data) -{ - return -ENODEV; -} - -static inline int iommu_unregister_device_fault_handler(struct device *dev) -{ - return 0; -} - -static inline -int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) -{ - return -ENODEV; -} - -static inline int iommu_page_response(struct device *dev, - struct iommu_page_response *msg) -{ - return -ENODEV; -} - static inline int iommu_group_id(struct iommu_group *group) { return -ENODEV; @@ -1256,7 +1329,7 @@ static inline int iommu_fwspec_add_ids(struct device *dev, u32 *ids, } static inline -const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode) +const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode) { return NULL; } @@ -1311,12 +1384,6 @@ static inline int iommu_device_claim_dma_owner(struct device *dev, void *owner) return -ENODEV; } -static inline struct iommu_domain * -iommu_sva_domain_alloc(struct device *dev, struct mm_struct *mm) -{ - return NULL; -} - static inline int iommu_attach_device_pasid(struct iommu_domain *domain, struct device *dev, ioasid_t pasid) { @@ -1343,6 +1410,14 @@ static inline ioasid_t iommu_alloc_global_pasid(struct device *dev) static inline void iommu_free_global_pasid(ioasid_t pasid) {} #endif /* CONFIG_IOMMU_API */ +#if IS_ENABLED(CONFIG_LOCKDEP) && IS_ENABLED(CONFIG_IOMMU_API) +void iommu_group_mutex_assert(struct device *dev); +#else +static inline void iommu_group_mutex_assert(struct device *dev) +{ +} +#endif + /** * iommu_map_sgtable - Map the given buffer to the IOMMU domain * @domain: The IOMMU domain to perform the mapping @@ -1456,6 +1531,8 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm); void iommu_sva_unbind_device(struct iommu_sva *handle); u32 iommu_sva_get_pasid(struct iommu_sva *handle); +struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, + struct mm_struct *mm); #else static inline struct iommu_sva * iommu_sva_bind_device(struct device *dev, struct mm_struct *mm) @@ -1480,6 +1557,68 @@ static inline u32 mm_get_enqcmd_pasid(struct mm_struct *mm) } static inline void mm_pasid_drop(struct mm_struct *mm) {} + +static inline struct iommu_domain * +iommu_sva_domain_alloc(struct device *dev, struct mm_struct *mm) +{ + return NULL; +} #endif /* CONFIG_IOMMU_SVA */ +#ifdef CONFIG_IOMMU_IOPF +int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev); +void iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev); +int iopf_queue_flush_dev(struct device *dev); +struct iopf_queue *iopf_queue_alloc(const char *name); +void iopf_queue_free(struct iopf_queue *queue); +int iopf_queue_discard_partial(struct iopf_queue *queue); +void iopf_free_group(struct iopf_group *group); +void iommu_report_device_fault(struct device *dev, struct iopf_fault *evt); +void iopf_group_response(struct iopf_group *group, + enum iommu_page_response_code status); +#else +static inline int +iopf_queue_add_device(struct iopf_queue *queue, struct device *dev) +{ + return -ENODEV; +} + +static inline void +iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev) +{ +} + +static inline int iopf_queue_flush_dev(struct device *dev) +{ + return -ENODEV; +} + +static inline struct iopf_queue *iopf_queue_alloc(const char *name) +{ + return NULL; +} + +static inline void iopf_queue_free(struct iopf_queue *queue) +{ +} + +static inline int iopf_queue_discard_partial(struct iopf_queue *queue) +{ + return -ENODEV; +} + +static inline void iopf_free_group(struct iopf_group *group) +{ +} + +static inline void +iommu_report_device_fault(struct device *dev, struct iopf_fault *evt) +{ +} + +static inline void iopf_group_response(struct iopf_group *group, + enum iommu_page_response_code status) +{ +} +#endif /* CONFIG_IOMMU_IOPF */ #endif /* __LINUX_IOMMU_H */ diff --git a/include/linux/kasan.h b/include/linux/kasan.h index dbb06d789e74..70d6a8f6e25d 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -429,7 +429,6 @@ struct kasan_cache { }; size_t kasan_metadata_size(struct kmem_cache *cache, bool in_object); -slab_flags_t kasan_never_merge(void); void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, slab_flags_t *flags); @@ -446,11 +445,6 @@ static inline size_t kasan_metadata_size(struct kmem_cache *cache, { return 0; } -/* And thus nothing prevents cache merging. */ -static inline slab_flags_t kasan_never_merge(void) -{ - return 0; -} /* And no cache-related metadata initialization is required. */ static inline void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 642272576582..a8057a3f8de6 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -94,6 +94,8 @@ LSM_HOOK(int, 0, path_mkdir, const struct path *dir, struct dentry *dentry, LSM_HOOK(int, 0, path_rmdir, const struct path *dir, struct dentry *dentry) LSM_HOOK(int, 0, path_mknod, const struct path *dir, struct dentry *dentry, umode_t mode, unsigned int dev) +LSM_HOOK(void, LSM_RET_VOID, path_post_mknod, struct mnt_idmap *idmap, + struct dentry *dentry) LSM_HOOK(int, 0, path_truncate, const struct path *path) LSM_HOOK(int, 0, path_symlink, const struct path *dir, struct dentry *dentry, const char *old_name) @@ -119,6 +121,8 @@ LSM_HOOK(int, 0, inode_init_security_anon, struct inode *inode, const struct qstr *name, const struct inode *context_inode) LSM_HOOK(int, 0, inode_create, struct inode *dir, struct dentry *dentry, umode_t mode) +LSM_HOOK(void, LSM_RET_VOID, inode_post_create_tmpfile, struct mnt_idmap *idmap, + struct inode *inode) LSM_HOOK(int, 0, inode_link, struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry) LSM_HOOK(int, 0, inode_unlink, struct inode *dir, struct dentry *dentry) @@ -135,7 +139,10 @@ LSM_HOOK(int, 0, inode_readlink, struct dentry *dentry) LSM_HOOK(int, 0, inode_follow_link, struct dentry *dentry, struct inode *inode, bool rcu) LSM_HOOK(int, 0, inode_permission, struct inode *inode, int mask) -LSM_HOOK(int, 0, inode_setattr, struct dentry *dentry, struct iattr *attr) +LSM_HOOK(int, 0, inode_setattr, struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +LSM_HOOK(void, LSM_RET_VOID, inode_post_setattr, struct mnt_idmap *idmap, + struct dentry *dentry, int ia_valid) LSM_HOOK(int, 0, inode_getattr, const struct path *path) LSM_HOOK(int, 0, inode_setxattr, struct mnt_idmap *idmap, struct dentry *dentry, const char *name, const void *value, @@ -146,12 +153,18 @@ LSM_HOOK(int, 0, inode_getxattr, struct dentry *dentry, const char *name) LSM_HOOK(int, 0, inode_listxattr, struct dentry *dentry) LSM_HOOK(int, 0, inode_removexattr, struct mnt_idmap *idmap, struct dentry *dentry, const char *name) +LSM_HOOK(void, LSM_RET_VOID, inode_post_removexattr, struct dentry *dentry, + const char *name) LSM_HOOK(int, 0, inode_set_acl, struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl) +LSM_HOOK(void, LSM_RET_VOID, inode_post_set_acl, struct dentry *dentry, + const char *acl_name, struct posix_acl *kacl) LSM_HOOK(int, 0, inode_get_acl, struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) LSM_HOOK(int, 0, inode_remove_acl, struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) +LSM_HOOK(void, LSM_RET_VOID, inode_post_remove_acl, struct mnt_idmap *idmap, + struct dentry *dentry, const char *acl_name) LSM_HOOK(int, 0, inode_need_killpriv, struct dentry *dentry) LSM_HOOK(int, 0, inode_killpriv, struct mnt_idmap *idmap, struct dentry *dentry) @@ -168,6 +181,7 @@ LSM_HOOK(int, 0, kernfs_init_security, struct kernfs_node *kn_dir, struct kernfs_node *kn) LSM_HOOK(int, 0, file_permission, struct file *file, int mask) LSM_HOOK(int, 0, file_alloc_security, struct file *file) +LSM_HOOK(void, LSM_RET_VOID, file_release, struct file *file) LSM_HOOK(void, LSM_RET_VOID, file_free_security, struct file *file) LSM_HOOK(int, 0, file_ioctl, struct file *file, unsigned int cmd, unsigned long arg) @@ -186,6 +200,7 @@ LSM_HOOK(int, 0, file_send_sigiotask, struct task_struct *tsk, struct fown_struct *fown, int sig) LSM_HOOK(int, 0, file_receive, struct file *file) LSM_HOOK(int, 0, file_open, struct file *file) +LSM_HOOK(int, 0, file_post_open, struct file *file, int mask) LSM_HOOK(int, 0, file_truncate, struct file *file) LSM_HOOK(int, 0, task_alloc, struct task_struct *task, unsigned long clone_flags) @@ -390,6 +405,9 @@ LSM_HOOK(void, LSM_RET_VOID, key_free, struct key *key) LSM_HOOK(int, 0, key_permission, key_ref_t key_ref, const struct cred *cred, enum key_need_perm need_perm) LSM_HOOK(int, 0, key_getsecurity, struct key *key, char **buffer) +LSM_HOOK(void, LSM_RET_VOID, key_post_create_or_update, struct key *keyring, + struct key *key, const void *payload, size_t payload_len, + unsigned long flags, bool create) #endif /* CONFIG_KEYS */ #ifdef CONFIG_AUDIT diff --git a/include/linux/mfd/da9055/pdata.h b/include/linux/mfd/da9055/pdata.h index d3f126990ad0..137a2b067512 100644 --- a/include/linux/mfd/da9055/pdata.h +++ b/include/linux/mfd/da9055/pdata.h @@ -7,7 +7,6 @@ #define DA9055_MAX_REGULATORS 8 struct da9055; -struct gpio_desc; enum gpio_select { NO_GPIO = 0, @@ -23,16 +22,6 @@ struct da9055_pdata { struct regulator_init_data *regulators[DA9055_MAX_REGULATORS]; /* Enable RTC in RESET Mode */ bool reset_enable; - /* - * GPI muxed pin to control - * regulator state A/B, 0 if not available. - */ - int *gpio_ren; - /* - * GPI muxed pin to control - * regulator set, 0 if not available. - */ - int *gpio_rsel; /* * Regulator mode control bits value (GPI offset) that * controls the regulator state, 0 if not available. @@ -43,7 +32,5 @@ struct da9055_pdata { * controls the regulator set A/B, 0 if not available. */ enum gpio_select *reg_rsel; - /* GPIO descriptors to enable regulator, NULL if not available */ - struct gpio_desc **ena_gpiods; }; #endif /* __DA9055_PDATA_H */ diff --git a/include/linux/mfd/lp8788.h b/include/linux/mfd/lp8788.h index 3d5c480d58ea..51b47966a04d 100644 --- a/include/linux/mfd/lp8788.h +++ b/include/linux/mfd/lp8788.h @@ -10,7 +10,6 @@ #ifndef __MFD_LP8788_H__ #define __MFD_LP8788_H__ -#include #include #include #include @@ -159,21 +158,17 @@ struct lp8788; /* * lp8788_buck1_dvs - * @gpio : gpio pin number for dvs control * @vsel : dvs selector for buck v1 register */ struct lp8788_buck1_dvs { - int gpio; enum lp8788_dvs_sel vsel; }; /* * lp8788_buck2_dvs - * @gpio : two gpio pin numbers are used for dvs * @vsel : dvs selector for buck v2 register */ struct lp8788_buck2_dvs { - int gpio[LP8788_NUM_BUCK2_DVS]; enum lp8788_dvs_sel vsel; }; @@ -268,8 +263,8 @@ struct lp8788_vib_platform_data { * @buck_data : regulator initial data for buck * @dldo_data : regulator initial data for digital ldo * @aldo_data : regulator initial data for analog ldo - * @buck1_dvs : gpio configurations for buck1 dvs - * @buck2_dvs : gpio configurations for buck2 dvs + * @buck1_dvs : configurations for buck1 dvs + * @buck2_dvs : configurations for buck2 dvs * @chg_pdata : platform data for charger driver * @alarm_sel : rtc alarm selection (1 or 2) * @bl_pdata : configurable data for backlight driver diff --git a/include/linux/mfd/max8997.h b/include/linux/mfd/max8997.h index 6193905abbb5..5c2cc1103437 100644 --- a/include/linux/mfd/max8997.h +++ b/include/linux/mfd/max8997.h @@ -178,7 +178,6 @@ struct max8997_platform_data { * */ bool ignore_gpiodvs_side_effect; - int buck125_gpios[3]; /* GPIO of [0]SET1, [1]SET2, [2]SET3 */ int buck125_default_idx; /* Default value of SET1, 2, 3 */ unsigned int buck1_voltage[8]; /* buckx_voltage in uV */ bool buck1_gpiodvs; diff --git a/include/linux/mfd/max8998.h b/include/linux/mfd/max8998.h index 79c020bd0c70..a054e55c8646 100644 --- a/include/linux/mfd/max8998.h +++ b/include/linux/mfd/max8998.h @@ -65,10 +65,7 @@ struct max8998_regulator_data { * be other than the preset values. * @buck1_voltage: BUCK1 DVS mode 1 voltage registers * @buck2_voltage: BUCK2 DVS mode 2 voltage registers - * @buck1_set1: BUCK1 gpio pin 1 to set output voltage - * @buck1_set2: BUCK1 gpio pin 2 to set output voltage * @buck1_default_idx: Default for BUCK1 gpio pin 1, 2 - * @buck2_set3: BUCK2 gpio pin to set output voltage * @buck2_default_idx: Default for BUCK2 gpio pin. * @wakeup: Allow to wake up from suspend * @rtc_delay: LP3974 RTC chip bug that requires delay after a register @@ -91,10 +88,7 @@ struct max8998_platform_data { bool buck_voltage_lock; int buck1_voltage[4]; int buck2_voltage[2]; - int buck1_set1; - int buck1_set2; int buck1_default_idx; - int buck2_set3; int buck2_default_idx; bool wakeup; bool rtc_delay; diff --git a/include/linux/mmc/host.h b/include/linux/mmc/host.h index 2f445c651742..5894bf912f7b 100644 --- a/include/linux/mmc/host.h +++ b/include/linux/mmc/host.h @@ -539,7 +539,7 @@ struct mmc_host *devm_mmc_alloc_host(struct device *dev, int extra); int mmc_add_host(struct mmc_host *); void mmc_remove_host(struct mmc_host *); void mmc_free_host(struct mmc_host *); -void mmc_of_parse_clk_phase(struct mmc_host *host, +void mmc_of_parse_clk_phase(struct device *dev, struct mmc_clk_phase_map *map); int mmc_of_parse(struct mmc_host *host); int mmc_of_parse_voltage(struct mmc_host *host, u32 *mask); diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h index f458469c5ce5..7a9a07ea451b 100644 --- a/include/linux/mod_devicetable.h +++ b/include/linux/mod_devicetable.h @@ -960,4 +960,14 @@ struct vchiq_device_id { char name[32]; }; +/** + * struct coreboot_device_id - Identifies a coreboot table entry + * @tag: tag ID + * @driver_data: driver specific data + */ +struct coreboot_device_id { + __u32 tag; + kernel_ulong_t driver_data; +}; + #endif /* LINUX_MOD_DEVICETABLE_H */ diff --git a/include/linux/pci.h b/include/linux/pci.h index 7ab0d13672da..213109d3c601 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -2517,6 +2517,11 @@ static inline struct pci_dev *pcie_find_root_port(struct pci_dev *dev) return NULL; } +static inline bool pci_dev_is_disconnected(const struct pci_dev *dev) +{ + return dev->error_state == pci_channel_io_perm_failure; +} + void pci_request_acs(void); bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags); bool pci_acs_path_enabled(struct pci_dev *start, diff --git a/include/linux/platform_data/x86/pwm-lpss.h b/include/linux/platform_data/x86/pwm-lpss.h index c852fe24fe2a..752c06b47cc8 100644 --- a/include/linux/platform_data/x86/pwm-lpss.h +++ b/include/linux/platform_data/x86/pwm-lpss.h @@ -27,7 +27,7 @@ struct pwm_lpss_boardinfo { bool other_devices_aml_touches_pwm_regs; }; -struct pwm_lpss_chip *devm_pwm_lpss_probe(struct device *dev, void __iomem *base, - const struct pwm_lpss_boardinfo *info); +struct pwm_chip *devm_pwm_lpss_probe(struct device *dev, void __iomem *base, + const struct pwm_lpss_boardinfo *info); #endif /* __PLATFORM_DATA_X86_PWM_LPSS_H */ diff --git a/include/linux/pm.h b/include/linux/pm.h index a2f3e53a8196..97b0e23363c8 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -662,8 +662,8 @@ struct pm_subsys_data { struct dev_pm_info { pm_message_t power_state; - unsigned int can_wakeup:1; - unsigned int async_suspend:1; + bool can_wakeup:1; + bool async_suspend:1; bool in_dpm_list:1; /* Owned by the PM core */ bool is_prepared:1; /* Owned by the PM core */ bool is_suspended:1; /* Ditto */ @@ -682,10 +682,10 @@ struct dev_pm_info { bool syscore:1; bool no_pm_callbacks:1; /* Owned by the PM core */ bool async_in_progress:1; /* Owned by the PM core */ - unsigned int must_resume:1; /* Owned by the PM core */ - unsigned int may_skip_resume:1; /* Set by subsystems */ + bool must_resume:1; /* Owned by the PM core */ + bool may_skip_resume:1; /* Set by subsystems */ #else - unsigned int should_wakeup:1; + bool should_wakeup:1; #endif #ifdef CONFIG_PM struct hrtimer suspend_timer; @@ -696,17 +696,17 @@ struct dev_pm_info { atomic_t usage_count; atomic_t child_count; unsigned int disable_depth:3; - unsigned int idle_notification:1; - unsigned int request_pending:1; - unsigned int deferred_resume:1; - unsigned int needs_force_resume:1; - unsigned int runtime_auto:1; + bool idle_notification:1; + bool request_pending:1; + bool deferred_resume:1; + bool needs_force_resume:1; + bool runtime_auto:1; bool ignore_children:1; - unsigned int no_callbacks:1; - unsigned int irq_safe:1; - unsigned int use_autosuspend:1; - unsigned int timer_autosuspends:1; - unsigned int memalloc_noio:1; + bool no_callbacks:1; + bool irq_safe:1; + bool use_autosuspend:1; + bool timer_autosuspends:1; + bool memalloc_noio:1; unsigned int links_count; enum rpm_request request; enum rpm_status runtime_status; diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h index b97c5e9820f9..772d3280d35f 100644 --- a/include/linux/pm_domain.h +++ b/include/linux/pm_domain.h @@ -19,6 +19,33 @@ #include #include +/* + * Flags to control the behaviour when attaching a device to its PM domains. + * + * PD_FLAG_NO_DEV_LINK: As the default behaviour creates a device-link + * for every PM domain that gets attached, this + * flag can be used to skip that. + * + * PD_FLAG_DEV_LINK_ON: Add the DL_FLAG_RPM_ACTIVE to power-on the + * supplier and its PM domain when creating the + * device-links. + * + */ +#define PD_FLAG_NO_DEV_LINK BIT(0) +#define PD_FLAG_DEV_LINK_ON BIT(1) + +struct dev_pm_domain_attach_data { + const char * const *pd_names; + const u32 num_pd_names; + const u32 pd_flags; +}; + +struct dev_pm_domain_list { + struct device **pd_devs; + struct device_link **pd_links; + u32 num_pds; +}; + /* * Flags to control the behaviour of a genpd. * @@ -322,7 +349,7 @@ static inline void dev_pm_genpd_resume(struct device *dev) {} /* OF PM domain providers */ struct of_device_id; -typedef struct generic_pm_domain *(*genpd_xlate_t)(struct of_phandle_args *args, +typedef struct generic_pm_domain *(*genpd_xlate_t)(const struct of_phandle_args *args, void *data); struct genpd_onecell_data { @@ -337,11 +364,11 @@ int of_genpd_add_provider_simple(struct device_node *np, int of_genpd_add_provider_onecell(struct device_node *np, struct genpd_onecell_data *data); void of_genpd_del_provider(struct device_node *np); -int of_genpd_add_device(struct of_phandle_args *args, struct device *dev); -int of_genpd_add_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec); -int of_genpd_remove_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec); +int of_genpd_add_device(const struct of_phandle_args *args, struct device *dev); +int of_genpd_add_subdomain(const struct of_phandle_args *parent_spec, + const struct of_phandle_args *subdomain_spec); +int of_genpd_remove_subdomain(const struct of_phandle_args *parent_spec, + const struct of_phandle_args *subdomain_spec); struct generic_pm_domain *of_genpd_remove_last(struct device_node *np); int of_genpd_parse_idle_states(struct device_node *dn, struct genpd_power_state **states, int *n); @@ -366,20 +393,20 @@ static inline int of_genpd_add_provider_onecell(struct device_node *np, static inline void of_genpd_del_provider(struct device_node *np) {} -static inline int of_genpd_add_device(struct of_phandle_args *args, +static inline int of_genpd_add_device(const struct of_phandle_args *args, struct device *dev) { return -ENODEV; } -static inline int of_genpd_add_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec) +static inline int of_genpd_add_subdomain(const struct of_phandle_args *parent_spec, + const struct of_phandle_args *subdomain_spec) { return -ENODEV; } -static inline int of_genpd_remove_subdomain(struct of_phandle_args *parent_spec, - struct of_phandle_args *subdomain_spec) +static inline int of_genpd_remove_subdomain(const struct of_phandle_args *parent_spec, + const struct of_phandle_args *subdomain_spec) { return -ENODEV; } @@ -420,7 +447,11 @@ struct device *dev_pm_domain_attach_by_id(struct device *dev, unsigned int index); struct device *dev_pm_domain_attach_by_name(struct device *dev, const char *name); +int dev_pm_domain_attach_list(struct device *dev, + const struct dev_pm_domain_attach_data *data, + struct dev_pm_domain_list **list); void dev_pm_domain_detach(struct device *dev, bool power_off); +void dev_pm_domain_detach_list(struct dev_pm_domain_list *list); int dev_pm_domain_start(struct device *dev); void dev_pm_domain_set(struct device *dev, struct dev_pm_domain *pd); int dev_pm_domain_set_performance_state(struct device *dev, unsigned int state); @@ -439,7 +470,14 @@ static inline struct device *dev_pm_domain_attach_by_name(struct device *dev, { return NULL; } +static inline int dev_pm_domain_attach_list(struct device *dev, + const struct dev_pm_domain_attach_data *data, + struct dev_pm_domain_list **list) +{ + return 0; +} static inline void dev_pm_domain_detach(struct device *dev, bool power_off) {} +static inline void dev_pm_domain_detach_list(struct dev_pm_domain_list *list) {} static inline int dev_pm_domain_start(struct device *dev) { return 0; diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 76dcb7f37bcd..065a47382302 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -16,6 +16,7 @@ #include struct clk; +struct cpufreq_frequency_table; struct regulator; struct dev_pm_opp; struct device; @@ -87,12 +88,14 @@ struct dev_pm_opp_config { /** * struct dev_pm_opp_data - The data to use to initialize an OPP. + * @turbo: Flag to indicate whether the OPP is to be marked turbo or not. * @level: The performance level for the OPP. Set level to OPP_LEVEL_UNSET if * level field isn't used. * @freq: The clock rate in Hz for the OPP. * @u_volt: The voltage in uV for the OPP. */ struct dev_pm_opp_data { + bool turbo; unsigned int level; unsigned long freq; unsigned long u_volt; @@ -444,6 +447,21 @@ static inline int dev_pm_opp_sync_regulators(struct device *dev) #endif /* CONFIG_PM_OPP */ +#if defined(CONFIG_CPU_FREQ) && defined(CONFIG_PM_OPP) +int dev_pm_opp_init_cpufreq_table(struct device *dev, struct cpufreq_frequency_table **table); +void dev_pm_opp_free_cpufreq_table(struct device *dev, struct cpufreq_frequency_table **table); +#else +static inline int dev_pm_opp_init_cpufreq_table(struct device *dev, struct cpufreq_frequency_table **table) +{ + return -EINVAL; +} + +static inline void dev_pm_opp_free_cpufreq_table(struct device *dev, struct cpufreq_frequency_table **table) +{ +} +#endif + + #if defined(CONFIG_PM_OPP) && defined(CONFIG_OF) int dev_pm_opp_of_add_table(struct device *dev); int dev_pm_opp_of_add_table_indexed(struct device *dev, int index); diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h index 7c9b35448563..d39dc863f612 100644 --- a/include/linux/pm_runtime.h +++ b/include/linux/pm_runtime.h @@ -72,7 +72,8 @@ extern int pm_runtime_force_resume(struct device *dev); extern int __pm_runtime_idle(struct device *dev, int rpmflags); extern int __pm_runtime_suspend(struct device *dev, int rpmflags); extern int __pm_runtime_resume(struct device *dev, int rpmflags); -extern int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count); +extern int pm_runtime_get_if_active(struct device *dev); +extern int pm_runtime_get_if_in_use(struct device *dev); extern int pm_schedule_suspend(struct device *dev, unsigned int delay); extern int __pm_runtime_set_status(struct device *dev, unsigned int status); extern int pm_runtime_barrier(struct device *dev); @@ -94,18 +95,6 @@ extern void pm_runtime_release_supplier(struct device_link *link); extern int devm_pm_runtime_enable(struct device *dev); -/** - * pm_runtime_get_if_in_use - Conditionally bump up runtime PM usage counter. - * @dev: Target device. - * - * Increment the runtime PM usage counter of @dev if its runtime PM status is - * %RPM_ACTIVE and its runtime PM usage counter is greater than 0. - */ -static inline int pm_runtime_get_if_in_use(struct device *dev) -{ - return pm_runtime_get_if_active(dev, false); -} - /** * pm_suspend_ignore_children - Set runtime PM behavior regarding children. * @dev: Target device. @@ -275,8 +264,7 @@ static inline int pm_runtime_get_if_in_use(struct device *dev) { return -EINVAL; } -static inline int pm_runtime_get_if_active(struct device *dev, - bool ign_usage_count) +static inline int pm_runtime_get_if_active(struct device *dev) { return -EINVAL; } @@ -460,6 +448,18 @@ static inline int pm_runtime_put(struct device *dev) return __pm_runtime_idle(dev, RPM_GET_PUT | RPM_ASYNC); } +/** + * __pm_runtime_put_autosuspend - Drop device usage counter and queue autosuspend if 0. + * @dev: Target device. + * + * Decrement the runtime PM usage counter of @dev and if it turns out to be + * equal to 0, queue up a work item for @dev like in pm_request_autosuspend(). + */ +static inline int __pm_runtime_put_autosuspend(struct device *dev) +{ + return __pm_runtime_suspend(dev, RPM_GET_PUT | RPM_ASYNC | RPM_AUTO); +} + /** * pm_runtime_put_autosuspend - Drop device usage counter and queue autosuspend if 0. * @dev: Target device. diff --git a/include/linux/printk.h b/include/linux/printk.h index 8ef499ab3c1e..955e31860095 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -273,6 +273,8 @@ static inline void printk_trigger_flush(void) } #endif +bool this_cpu_in_panic(void); + #ifdef CONFIG_SMP extern int __printk_cpu_sync_try_get(void); extern void __printk_cpu_sync_wait(void); diff --git a/include/linux/pwm.h b/include/linux/pwm.h index fcc2c4496f73..4a6568dfdf3f 100644 --- a/include/linux/pwm.h +++ b/include/linux/pwm.h @@ -271,8 +271,8 @@ struct pwm_ops { * @id: unique number of this PWM chip * @npwm: number of PWMs controlled by this chip * @of_xlate: request a PWM device given a device tree PWM specifier - * @of_pwm_n_cells: number of cells expected in the device tree PWM specifier * @atomic: can the driver's ->apply() be called in atomic context + * @driver_data: Private pointer for driver specific info * @pwms: array of PWM devices allocated by the framework */ struct pwm_chip { @@ -284,13 +284,36 @@ struct pwm_chip { struct pwm_device * (*of_xlate)(struct pwm_chip *chip, const struct of_phandle_args *args); - unsigned int of_pwm_n_cells; bool atomic; /* only used internally by the PWM framework */ + void *driver_data; struct pwm_device *pwms; }; +static inline struct device *pwmchip_parent(const struct pwm_chip *chip) +{ + return chip->dev; +} + +static inline void *pwmchip_get_drvdata(struct pwm_chip *chip) +{ + /* + * After pwm_chip got a dedicated struct device, this can be replaced by + * dev_get_drvdata(&chip->dev); + */ + return chip->driver_data; +} + +static inline void pwmchip_set_drvdata(struct pwm_chip *chip, void *data) +{ + /* + * After pwm_chip got a dedicated struct device, this can be replaced by + * dev_set_drvdata(&chip->dev, data); + */ + chip->driver_data = data; +} + #if IS_ENABLED(CONFIG_PWM) /* PWM user APIs */ int pwm_apply_might_sleep(struct pwm_device *pwm, const struct pwm_state *state); @@ -380,6 +403,10 @@ static inline bool pwm_might_sleep(struct pwm_device *pwm) int pwm_capture(struct pwm_device *pwm, struct pwm_capture *result, unsigned long timeout); +void pwmchip_put(struct pwm_chip *chip); +struct pwm_chip *pwmchip_alloc(struct device *parent, unsigned int npwm, size_t sizeof_priv); +struct pwm_chip *devm_pwmchip_alloc(struct device *parent, unsigned int npwm, size_t sizeof_priv); + int __pwmchip_add(struct pwm_chip *chip, struct module *owner); #define pwmchip_add(chip) __pwmchip_add(chip, THIS_MODULE) void pwmchip_remove(struct pwm_chip *chip); @@ -452,6 +479,24 @@ static inline int pwm_capture(struct pwm_device *pwm, return -EINVAL; } +static inline void pwmchip_put(struct pwm_chip *chip) +{ +} + +static inline struct pwm_chip *pwmchip_alloc(struct device *parent, + unsigned int npwm, + size_t sizeof_priv) +{ + return ERR_PTR(-EINVAL); +} + +static inline struct pwm_chip *devm_pwmchip_alloc(struct device *parent, + unsigned int npwm, + size_t sizeof_priv) +{ + return pwmchip_alloc(parent, npwm, sizeof_priv); +} + static inline int pwmchip_add(struct pwm_chip *chip) { return -EINVAL; diff --git a/include/linux/regmap.h b/include/linux/regmap.h index c9182a47736e..b743241cfb7c 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -332,6 +332,10 @@ typedef void (*regmap_unlock)(void *); * @io_port: Support IO port accessors. Makes sense only when MMIO vs. IO port * access can be distinguished. * @max_register: Optional, specifies the maximum valid register address. + * @max_register_is_0: Optional, specifies that zero value in @max_register + * should be taken into account. This is a workaround to + * apply handling of @max_register for regmap that contains + * only one register. * @wr_table: Optional, points to a struct regmap_access_table specifying * valid ranges for write access. * @rd_table: As above, for read access. @@ -422,6 +426,7 @@ struct regmap_config { bool io_port; unsigned int max_register; + bool max_register_is_0; const struct regmap_access_table *wr_table; const struct regmap_access_table *rd_table; const struct regmap_access_table *volatile_table; diff --git a/include/linux/regulator/max8973-regulator.h b/include/linux/regulator/max8973-regulator.h index 8313e7ed6aec..a225e9eeb30d 100644 --- a/include/linux/regulator/max8973-regulator.h +++ b/include/linux/regulator/max8973-regulator.h @@ -48,10 +48,6 @@ * control signal from EN input pin. If it is false then * voltage output will be enabled/disabled through EN bit of * device register. - * @enable_gpio: Enable GPIO. If EN pin is controlled through GPIO from host - * then GPIO number can be provided. If no GPIO controlled then - * it should be -1. - * @dvs_gpio: GPIO for dvs. It should be -1 if this is tied with fixed logic. * @dvs_def_state: Default state of dvs. 1 if it is high else 0. */ struct max8973_regulator_platform_data { @@ -59,8 +55,6 @@ struct max8973_regulator_platform_data { unsigned long control_flags; unsigned long junction_temp_warning; bool enable_ext_control; - int enable_gpio; - int dvs_gpio; unsigned dvs_def_state:1; }; diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index 2ee94ff0320c..b807141acc14 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -140,6 +140,8 @@ struct scmi_perf_domain_info { * @level_set: sets the performance level of a domain * @level_get: gets the performance level of a domain * @transition_latency_get: gets the DVFS transition latency for a given device + * @rate_limit_get: gets the minimum time (us) required between successive + * requests * @device_opps_add: adds all the OPPs for a given device * @freq_set: sets the frequency for a given device using sustained frequency * to sustained performance level mapping @@ -149,6 +151,8 @@ struct scmi_perf_domain_info { * at a given frequency * @fast_switch_possible: indicates if fast DVFS switching is possible or not * for a given device + * @fast_switch_rate_limit: gets the minimum time (us) required between + * successive fast_switching requests * @power_scale_mw_get: indicates if the power values provided are in milliWatts * or in some other (abstract) scale */ @@ -166,6 +170,8 @@ struct scmi_perf_proto_ops { u32 *level, bool poll); int (*transition_latency_get)(const struct scmi_protocol_handle *ph, u32 domain); + int (*rate_limit_get)(const struct scmi_protocol_handle *ph, + u32 domain, u32 *rate_limit); int (*device_opps_add)(const struct scmi_protocol_handle *ph, struct device *dev, u32 domain); int (*freq_set)(const struct scmi_protocol_handle *ph, u32 domain, @@ -176,6 +182,8 @@ struct scmi_perf_proto_ops { unsigned long *rate, unsigned long *power); bool (*fast_switch_possible)(const struct scmi_protocol_handle *ph, u32 domain); + int (*fast_switch_rate_limit)(const struct scmi_protocol_handle *ph, + u32 domain, u32 *rate_limit); enum scmi_power_scale (*power_scale_get)(const struct scmi_protocol_handle *ph); }; diff --git a/include/linux/security.h b/include/linux/security.h index 15804af54f37..f249f5b9a9d7 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -345,6 +345,8 @@ int security_inode_init_security_anon(struct inode *inode, const struct qstr *name, const struct inode *context_inode); int security_inode_create(struct inode *dir, struct dentry *dentry, umode_t mode); +void security_inode_post_create_tmpfile(struct mnt_idmap *idmap, + struct inode *inode); int security_inode_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry); int security_inode_unlink(struct inode *dir, struct dentry *dentry); @@ -362,6 +364,8 @@ int security_inode_follow_link(struct dentry *dentry, struct inode *inode, int security_inode_permission(struct inode *inode, int mask); int security_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr); +void security_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + int ia_valid); int security_inode_getattr(const struct path *path); int security_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, const char *name, @@ -369,16 +373,22 @@ int security_inode_setxattr(struct mnt_idmap *idmap, int security_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl); +void security_inode_post_set_acl(struct dentry *dentry, const char *acl_name, + struct posix_acl *kacl); int security_inode_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name); int security_inode_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name); +void security_inode_post_remove_acl(struct mnt_idmap *idmap, + struct dentry *dentry, + const char *acl_name); void security_inode_post_setxattr(struct dentry *dentry, const char *name, const void *value, size_t size, int flags); int security_inode_getxattr(struct dentry *dentry, const char *name); int security_inode_listxattr(struct dentry *dentry); int security_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, const char *name); +void security_inode_post_removexattr(struct dentry *dentry, const char *name); int security_inode_need_killpriv(struct dentry *dentry); int security_inode_killpriv(struct mnt_idmap *idmap, struct dentry *dentry); int security_inode_getsecurity(struct mnt_idmap *idmap, @@ -393,6 +403,7 @@ int security_kernfs_init_security(struct kernfs_node *kn_dir, struct kernfs_node *kn); int security_file_permission(struct file *file, int mask); int security_file_alloc(struct file *file); +void security_file_release(struct file *file); void security_file_free(struct file *file); int security_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg); int security_file_ioctl_compat(struct file *file, unsigned int cmd, @@ -409,6 +420,7 @@ int security_file_send_sigiotask(struct task_struct *tsk, struct fown_struct *fown, int sig); int security_file_receive(struct file *file); int security_file_open(struct file *file); +int security_file_post_open(struct file *file, int mask); int security_file_truncate(struct file *file); int security_task_alloc(struct task_struct *task, unsigned long clone_flags); void security_task_free(struct task_struct *task); @@ -807,6 +819,10 @@ static inline int security_inode_create(struct inode *dir, return 0; } +static inline void +security_inode_post_create_tmpfile(struct mnt_idmap *idmap, struct inode *inode) +{ } + static inline int security_inode_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry) @@ -880,6 +896,11 @@ static inline int security_inode_setattr(struct mnt_idmap *idmap, return 0; } +static inline void +security_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + int ia_valid) +{ } + static inline int security_inode_getattr(const struct path *path) { return 0; @@ -900,6 +921,11 @@ static inline int security_inode_set_acl(struct mnt_idmap *idmap, return 0; } +static inline void security_inode_post_set_acl(struct dentry *dentry, + const char *acl_name, + struct posix_acl *kacl) +{ } + static inline int security_inode_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) @@ -914,6 +940,11 @@ static inline int security_inode_remove_acl(struct mnt_idmap *idmap, return 0; } +static inline void security_inode_post_remove_acl(struct mnt_idmap *idmap, + struct dentry *dentry, + const char *acl_name) +{ } + static inline void security_inode_post_setxattr(struct dentry *dentry, const char *name, const void *value, size_t size, int flags) { } @@ -936,6 +967,10 @@ static inline int security_inode_removexattr(struct mnt_idmap *idmap, return cap_inode_removexattr(idmap, dentry, name); } +static inline void security_inode_post_removexattr(struct dentry *dentry, + const char *name) +{ } + static inline int security_inode_need_killpriv(struct dentry *dentry) { return cap_inode_need_killpriv(dentry); @@ -996,6 +1031,9 @@ static inline int security_file_alloc(struct file *file) return 0; } +static inline void security_file_release(struct file *file) +{ } + static inline void security_file_free(struct file *file) { } @@ -1063,6 +1101,11 @@ static inline int security_file_open(struct file *file) return 0; } +static inline int security_file_post_open(struct file *file, int mask) +{ + return 0; +} + static inline int security_file_truncate(struct file *file) { return 0; @@ -1872,6 +1915,7 @@ int security_path_mkdir(const struct path *dir, struct dentry *dentry, umode_t m int security_path_rmdir(const struct path *dir, struct dentry *dentry); int security_path_mknod(const struct path *dir, struct dentry *dentry, umode_t mode, unsigned int dev); +void security_path_post_mknod(struct mnt_idmap *idmap, struct dentry *dentry); int security_path_truncate(const struct path *path); int security_path_symlink(const struct path *dir, struct dentry *dentry, const char *old_name); @@ -1906,6 +1950,10 @@ static inline int security_path_mknod(const struct path *dir, struct dentry *den return 0; } +static inline void security_path_post_mknod(struct mnt_idmap *idmap, + struct dentry *dentry) +{ } + static inline int security_path_truncate(const struct path *path) { return 0; @@ -1957,6 +2005,9 @@ void security_key_free(struct key *key); int security_key_permission(key_ref_t key_ref, const struct cred *cred, enum key_need_perm need_perm); int security_key_getsecurity(struct key *key, char **_buffer); +void security_key_post_create_or_update(struct key *keyring, struct key *key, + const void *payload, size_t payload_len, + unsigned long flags, bool create); #else @@ -1984,6 +2035,14 @@ static inline int security_key_getsecurity(struct key *key, char **_buffer) return 0; } +static inline void security_key_post_create_or_update(struct key *keyring, + struct key *key, + const void *payload, + size_t payload_len, + unsigned long flags, + bool create) +{ } + #endif #endif /* CONFIG_KEYS */ diff --git a/include/linux/slab.h b/include/linux/slab.h index db1da6e05e05..4dfca434a798 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -21,29 +21,69 @@ #include #include +enum _slab_flag_bits { + _SLAB_CONSISTENCY_CHECKS, + _SLAB_RED_ZONE, + _SLAB_POISON, + _SLAB_KMALLOC, + _SLAB_HWCACHE_ALIGN, + _SLAB_CACHE_DMA, + _SLAB_CACHE_DMA32, + _SLAB_STORE_USER, + _SLAB_PANIC, + _SLAB_TYPESAFE_BY_RCU, + _SLAB_TRACE, +#ifdef CONFIG_DEBUG_OBJECTS + _SLAB_DEBUG_OBJECTS, +#endif + _SLAB_NOLEAKTRACE, + _SLAB_NO_MERGE, +#ifdef CONFIG_FAILSLAB + _SLAB_FAILSLAB, +#endif +#ifdef CONFIG_MEMCG_KMEM + _SLAB_ACCOUNT, +#endif +#ifdef CONFIG_KASAN_GENERIC + _SLAB_KASAN, +#endif + _SLAB_NO_USER_FLAGS, +#ifdef CONFIG_KFENCE + _SLAB_SKIP_KFENCE, +#endif +#ifndef CONFIG_SLUB_TINY + _SLAB_RECLAIM_ACCOUNT, +#endif + _SLAB_OBJECT_POISON, + _SLAB_CMPXCHG_DOUBLE, + _SLAB_FLAGS_LAST_BIT +}; + +#define __SLAB_FLAG_BIT(nr) ((slab_flags_t __force)(1U << (nr))) +#define __SLAB_FLAG_UNUSED ((slab_flags_t __force)(0U)) /* * Flags to pass to kmem_cache_create(). * The ones marked DEBUG need CONFIG_SLUB_DEBUG enabled, otherwise are no-op */ /* DEBUG: Perform (expensive) checks on alloc/free */ -#define SLAB_CONSISTENCY_CHECKS ((slab_flags_t __force)0x00000100U) +#define SLAB_CONSISTENCY_CHECKS __SLAB_FLAG_BIT(_SLAB_CONSISTENCY_CHECKS) /* DEBUG: Red zone objs in a cache */ -#define SLAB_RED_ZONE ((slab_flags_t __force)0x00000400U) +#define SLAB_RED_ZONE __SLAB_FLAG_BIT(_SLAB_RED_ZONE) /* DEBUG: Poison objects */ -#define SLAB_POISON ((slab_flags_t __force)0x00000800U) +#define SLAB_POISON __SLAB_FLAG_BIT(_SLAB_POISON) /* Indicate a kmalloc slab */ -#define SLAB_KMALLOC ((slab_flags_t __force)0x00001000U) +#define SLAB_KMALLOC __SLAB_FLAG_BIT(_SLAB_KMALLOC) /* Align objs on cache lines */ -#define SLAB_HWCACHE_ALIGN ((slab_flags_t __force)0x00002000U) +#define SLAB_HWCACHE_ALIGN __SLAB_FLAG_BIT(_SLAB_HWCACHE_ALIGN) /* Use GFP_DMA memory */ -#define SLAB_CACHE_DMA ((slab_flags_t __force)0x00004000U) +#define SLAB_CACHE_DMA __SLAB_FLAG_BIT(_SLAB_CACHE_DMA) /* Use GFP_DMA32 memory */ -#define SLAB_CACHE_DMA32 ((slab_flags_t __force)0x00008000U) +#define SLAB_CACHE_DMA32 __SLAB_FLAG_BIT(_SLAB_CACHE_DMA32) /* DEBUG: Store the last owner for bug hunting */ -#define SLAB_STORE_USER ((slab_flags_t __force)0x00010000U) +#define SLAB_STORE_USER __SLAB_FLAG_BIT(_SLAB_STORE_USER) /* Panic if kmem_cache_create() fails */ -#define SLAB_PANIC ((slab_flags_t __force)0x00040000U) +#define SLAB_PANIC __SLAB_FLAG_BIT(_SLAB_PANIC) /* * SLAB_TYPESAFE_BY_RCU - **WARNING** READ THIS! * @@ -95,21 +135,19 @@ * Note that SLAB_TYPESAFE_BY_RCU was originally named SLAB_DESTROY_BY_RCU. */ /* Defer freeing slabs to RCU */ -#define SLAB_TYPESAFE_BY_RCU ((slab_flags_t __force)0x00080000U) -/* Spread some memory over cpuset */ -#define SLAB_MEM_SPREAD ((slab_flags_t __force)0x00100000U) +#define SLAB_TYPESAFE_BY_RCU __SLAB_FLAG_BIT(_SLAB_TYPESAFE_BY_RCU) /* Trace allocations and frees */ -#define SLAB_TRACE ((slab_flags_t __force)0x00200000U) +#define SLAB_TRACE __SLAB_FLAG_BIT(_SLAB_TRACE) /* Flag to prevent checks on free */ #ifdef CONFIG_DEBUG_OBJECTS -# define SLAB_DEBUG_OBJECTS ((slab_flags_t __force)0x00400000U) +# define SLAB_DEBUG_OBJECTS __SLAB_FLAG_BIT(_SLAB_DEBUG_OBJECTS) #else -# define SLAB_DEBUG_OBJECTS 0 +# define SLAB_DEBUG_OBJECTS __SLAB_FLAG_UNUSED #endif /* Avoid kmemleak tracing */ -#define SLAB_NOLEAKTRACE ((slab_flags_t __force)0x00800000U) +#define SLAB_NOLEAKTRACE __SLAB_FLAG_BIT(_SLAB_NOLEAKTRACE) /* * Prevent merging with compatible kmem caches. This flag should be used @@ -121,25 +159,25 @@ * - performance critical caches, should be very rare and consulted with slab * maintainers, and not used together with CONFIG_SLUB_TINY */ -#define SLAB_NO_MERGE ((slab_flags_t __force)0x01000000U) +#define SLAB_NO_MERGE __SLAB_FLAG_BIT(_SLAB_NO_MERGE) /* Fault injection mark */ #ifdef CONFIG_FAILSLAB -# define SLAB_FAILSLAB ((slab_flags_t __force)0x02000000U) +# define SLAB_FAILSLAB __SLAB_FLAG_BIT(_SLAB_FAILSLAB) #else -# define SLAB_FAILSLAB 0 +# define SLAB_FAILSLAB __SLAB_FLAG_UNUSED #endif /* Account to memcg */ #ifdef CONFIG_MEMCG_KMEM -# define SLAB_ACCOUNT ((slab_flags_t __force)0x04000000U) +# define SLAB_ACCOUNT __SLAB_FLAG_BIT(_SLAB_ACCOUNT) #else -# define SLAB_ACCOUNT 0 +# define SLAB_ACCOUNT __SLAB_FLAG_UNUSED #endif #ifdef CONFIG_KASAN_GENERIC -#define SLAB_KASAN ((slab_flags_t __force)0x08000000U) +#define SLAB_KASAN __SLAB_FLAG_BIT(_SLAB_KASAN) #else -#define SLAB_KASAN 0 +#define SLAB_KASAN __SLAB_FLAG_UNUSED #endif /* @@ -147,20 +185,20 @@ * Intended for caches created for self-tests so they have only flags * specified in the code and other flags are ignored. */ -#define SLAB_NO_USER_FLAGS ((slab_flags_t __force)0x10000000U) +#define SLAB_NO_USER_FLAGS __SLAB_FLAG_BIT(_SLAB_NO_USER_FLAGS) #ifdef CONFIG_KFENCE -#define SLAB_SKIP_KFENCE ((slab_flags_t __force)0x20000000U) +#define SLAB_SKIP_KFENCE __SLAB_FLAG_BIT(_SLAB_SKIP_KFENCE) #else -#define SLAB_SKIP_KFENCE 0 +#define SLAB_SKIP_KFENCE __SLAB_FLAG_UNUSED #endif /* The following flags affect the page allocator grouping pages by mobility */ /* Objects are reclaimable */ #ifndef CONFIG_SLUB_TINY -#define SLAB_RECLAIM_ACCOUNT ((slab_flags_t __force)0x00020000U) +#define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_BIT(_SLAB_RECLAIM_ACCOUNT) #else -#define SLAB_RECLAIM_ACCOUNT ((slab_flags_t __force)0) +#define SLAB_RECLAIM_ACCOUNT __SLAB_FLAG_UNUSED #endif #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ diff --git a/include/linux/spi/pxa2xx_spi.h b/include/linux/spi/pxa2xx_spi.h index 0916cb9bcb0a..ca2cd4e30ead 100644 --- a/include/linux/spi/pxa2xx_spi.h +++ b/include/linux/spi/pxa2xx_spi.h @@ -5,6 +5,7 @@ #ifndef __LINUX_SPI_PXA2XX_SPI_H #define __LINUX_SPI_PXA2XX_SPI_H +#include #include #include @@ -22,7 +23,7 @@ struct pxa2xx_spi_controller { bool is_target; /* DMA engine specific config */ - bool (*dma_filter)(struct dma_chan *chan, void *param); + dma_filter_fn dma_filter; void *tx_param; void *rx_param; diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index 600fbd5daf68..b05d5a87c313 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -36,7 +36,7 @@ struct spi_message; * INTERFACES between SPI master-side drivers and SPI slave protocol handlers, * and SPI infrastructure. */ -extern struct bus_type spi_bus_type; +extern const struct bus_type spi_bus_type; /** * struct spi_statistics - statistics for spi transfers @@ -131,7 +131,6 @@ extern void spi_transfer_cs_change_delay_exec(struct spi_message *msg, * struct spi_device - Controller side proxy for an SPI slave device * @dev: Driver model representation of the device. * @controller: SPI controller used with the device. - * @master: Copy of controller, for backwards compatibility. * @max_speed_hz: Maximum clock rate to be used with this chip * (on this board); may be changed by the device's driver. * The spi_transfer.speed_hz can override this for each transfer. @@ -185,7 +184,6 @@ extern void spi_transfer_cs_change_delay_exec(struct spi_message *msg, struct spi_device { struct device dev; struct spi_controller *controller; - struct spi_controller *master; /* Compatibility layer */ u32 max_speed_hz; u8 chip_select[SPI_CS_CNT_MAX]; u8 bits_per_word; @@ -452,9 +450,11 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * the @cur_msg_completion. This flag is used to signal the context that * is running spi_finalize_current_message() that it needs to complete() * @cur_msg_mapped: message has been mapped for DMA + * @fallback: fallback to PIO if DMA transfer return failure with + * SPI_TRANS_FAIL_NO_START. + * @last_cs_mode_high: was (mode & SPI_CS_HIGH) true on the last call to set_cs. * @last_cs: the last chip_select that is recorded by set_cs, -1 on non chip * selected - * @last_cs_mode_high: was (mode & SPI_CS_HIGH) true on the last call to set_cs. * @xfer_completion: used by core transfer_one_message() * @busy: message pump is busy * @running: message pump is running @@ -477,6 +477,8 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * * @set_cs: set the logic level of the chip select line. May be called * from interrupt context. + * @optimize_message: optimize the message for reuse + * @unoptimize_message: release resources allocated by optimize_message * @prepare_message: set up the controller to transfer a single message, * for example doing DMA mapping. Called from threaded * context. @@ -529,8 +531,6 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * If the driver does not set this, the SPI core takes the snapshot as * close to the driver hand-over as possible. * @irq_flags: Interrupt enable state during PTP system timestamping - * @fallback: fallback to PIO if DMA transfer return failure with - * SPI_TRANS_FAIL_NO_START. * @queue_empty: signal green light for opportunistically skipping the queue * for spi_sync transfers. * @must_async: disable all fast paths in the core @@ -710,13 +710,15 @@ struct spi_controller { bool rt; bool auto_runtime_pm; bool cur_msg_mapped; - char last_cs[SPI_CS_CNT_MAX]; - char last_cs_index_mask; - bool last_cs_mode_high; bool fallback; + bool last_cs_mode_high; + s8 last_cs[SPI_CS_CNT_MAX]; + u32 last_cs_index_mask : SPI_CS_CNT_MAX; struct completion xfer_completion; size_t max_dma_len; + int (*optimize_message)(struct spi_message *msg); + int (*unoptimize_message)(struct spi_message *msg); int (*prepare_transfer_hardware)(struct spi_controller *ctlr); int (*transfer_one_message)(struct spi_controller *ctlr, struct spi_message *mesg); @@ -1113,16 +1115,19 @@ struct spi_transfer { * @spi: SPI device to which the transaction is queued * @is_dma_mapped: if true, the caller provided both DMA and CPU virtual * addresses for each transfer buffer + * @pre_optimized: peripheral driver pre-optimized the message + * @optimized: the message is in the optimized state + * @prepared: spi_prepare_message was called for the this message + * @status: zero for success, else negative errno * @complete: called to report transaction completions * @context: the argument to complete() when it's called * @frame_length: the total number of bytes in the message * @actual_length: the total number of bytes that were transferred in all * successful segments - * @status: zero for success, else negative errno * @queue: for use by whichever driver currently owns the message * @state: for use by whichever driver currently owns the message + * @opt_state: for use by whichever driver currently owns the message * @resources: for resource management when the SPI message is processed - * @prepared: spi_prepare_message was called for the this message * * A @spi_message is used to execute an atomic sequence of data transfers, * each represented by a struct spi_transfer. The sequence is "atomic" @@ -1145,6 +1150,11 @@ struct spi_message { unsigned is_dma_mapped:1; + /* spi_optimize_message() was called for this message */ + bool pre_optimized; + /* __spi_optimize_message() was called for this message */ + bool optimized; + /* spi_prepare_message() was called for this message */ bool prepared; @@ -1174,6 +1184,11 @@ struct spi_message { */ struct list_head queue; void *state; + /* + * Optional state for use by controller driver between calls to + * __spi_optimize_message() and __spi_unoptimize_message(). + */ + void *opt_state; /* List of spi_res resources when the SPI message is processed */ struct list_head resources; @@ -1257,6 +1272,9 @@ static inline void spi_message_free(struct spi_message *m) kfree(m); } +extern int spi_optimize_message(struct spi_device *spi, struct spi_message *msg); +extern void spi_unoptimize_message(struct spi_message *msg); + extern int spi_setup(struct spi_device *spi); extern int spi_async(struct spi_device *spi, struct spi_message *message); extern int spi_slave_abort(struct spi_device *spi); @@ -1298,7 +1316,7 @@ spi_max_transfer_size(struct spi_device *spi) */ static inline bool spi_is_bpw_supported(struct spi_device *spi, u32 bpw) { - u32 bpw_mask = spi->master->bits_per_word_mask; + u32 bpw_mask = spi->controller->bits_per_word_mask; if (bpw == 8 || (bpw <= 32 && bpw_mask & SPI_BPW_MASK(bpw))) return true; @@ -1365,12 +1383,10 @@ struct spi_replaced_transfers { extern int spi_split_transfers_maxsize(struct spi_controller *ctlr, struct spi_message *msg, - size_t maxsize, - gfp_t gfp); + size_t maxsize); extern int spi_split_transfers_maxwords(struct spi_controller *ctlr, struct spi_message *msg, - size_t maxwords, - gfp_t gfp); + size_t maxwords); /*---------------------------------------------------------------------------*/ @@ -1670,20 +1686,4 @@ spi_transfer_is_last(struct spi_controller *ctlr, struct spi_transfer *xfer) return list_is_last(&xfer->transfer_list, &ctlr->cur_msg->transfers); } -/* Compatibility layer */ -#define spi_master spi_controller - -#define spi_master_get_devdata(_ctlr) spi_controller_get_devdata(_ctlr) -#define spi_master_set_devdata(_ctlr, _data) \ - spi_controller_set_devdata(_ctlr, _data) -#define spi_master_get(_ctlr) spi_controller_get(_ctlr) -#define spi_master_put(_ctlr) spi_controller_put(_ctlr) -#define spi_master_suspend(_ctlr) spi_controller_suspend(_ctlr) -#define spi_master_resume(_ctlr) spi_controller_resume(_ctlr) - -#define spi_register_master(_ctlr) spi_register_controller(_ctlr) -#define devm_spi_register_master(_dev, _ctlr) \ - devm_spi_register_controller(_dev, _ctlr) -#define spi_unregister_master(_ctlr) spi_unregister_controller(_ctlr) - #endif /* __LINUX_SPI_H */ diff --git a/include/linux/spi/spi_bitbang.h b/include/linux/spi/spi_bitbang.h index 4444c2a992cb..b930eca2ef7b 100644 --- a/include/linux/spi/spi_bitbang.h +++ b/include/linux/spi/spi_bitbang.h @@ -10,7 +10,7 @@ struct spi_bitbang { u8 use_dma; u16 flags; /* extra spi->mode support */ - struct spi_master *master; + struct spi_controller *ctlr; /* setup_transfer() changes clock and/or wordsize to match settings * for this transfer; zeroes restore defaults from spi_device. diff --git a/include/linux/spi/spi_gpio.h b/include/linux/spi/spi_gpio.h index 9e7e83d8645b..5f0e1407917a 100644 --- a/include/linux/spi/spi_gpio.h +++ b/include/linux/spi/spi_gpio.h @@ -15,8 +15,8 @@ */ /** - * struct spi_gpio_platform_data - parameter for bitbanged SPI master - * @num_chipselect: how many slaves to allow + * struct spi_gpio_platform_data - parameter for bitbanged SPI host controller + * @num_chipselect: how many target devices to allow */ struct spi_gpio_platform_data { u16 num_chipselect; diff --git a/include/linux/suspend.h b/include/linux/suspend.h index b6067b93445d..e6d482ad44c8 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -40,65 +40,6 @@ typedef int __bitwise suspend_state_t; #define PM_SUSPEND_MIN PM_SUSPEND_TO_IDLE #define PM_SUSPEND_MAX ((__force suspend_state_t) 4) -enum suspend_stat_step { - SUSPEND_FREEZE = 1, - SUSPEND_PREPARE, - SUSPEND_SUSPEND, - SUSPEND_SUSPEND_LATE, - SUSPEND_SUSPEND_NOIRQ, - SUSPEND_RESUME_NOIRQ, - SUSPEND_RESUME_EARLY, - SUSPEND_RESUME -}; - -struct suspend_stats { - int success; - int fail; - int failed_freeze; - int failed_prepare; - int failed_suspend; - int failed_suspend_late; - int failed_suspend_noirq; - int failed_resume; - int failed_resume_early; - int failed_resume_noirq; -#define REC_FAILED_NUM 2 - int last_failed_dev; - char failed_devs[REC_FAILED_NUM][40]; - int last_failed_errno; - int errno[REC_FAILED_NUM]; - int last_failed_step; - u64 last_hw_sleep; - u64 total_hw_sleep; - u64 max_hw_sleep; - enum suspend_stat_step failed_steps[REC_FAILED_NUM]; -}; - -extern struct suspend_stats suspend_stats; - -static inline void dpm_save_failed_dev(const char *name) -{ - strscpy(suspend_stats.failed_devs[suspend_stats.last_failed_dev], - name, - sizeof(suspend_stats.failed_devs[0])); - suspend_stats.last_failed_dev++; - suspend_stats.last_failed_dev %= REC_FAILED_NUM; -} - -static inline void dpm_save_failed_errno(int err) -{ - suspend_stats.errno[suspend_stats.last_failed_errno] = err; - suspend_stats.last_failed_errno++; - suspend_stats.last_failed_errno %= REC_FAILED_NUM; -} - -static inline void dpm_save_failed_step(enum suspend_stat_step step) -{ - suspend_stats.failed_steps[suspend_stats.last_failed_step] = step; - suspend_stats.last_failed_step++; - suspend_stats.last_failed_step %= REC_FAILED_NUM; -} - /** * struct platform_suspend_ops - Callbacks for managing platform dependent * system sleep states. @@ -627,4 +568,19 @@ static inline void queue_up_suspend_work(void) {} #endif /* !CONFIG_PM_AUTOSLEEP */ +enum suspend_stat_step { + SUSPEND_WORKING = 0, + SUSPEND_FREEZE, + SUSPEND_PREPARE, + SUSPEND_SUSPEND, + SUSPEND_SUSPEND_LATE, + SUSPEND_SUSPEND_NOIRQ, + SUSPEND_RESUME_NOIRQ, + SUSPEND_RESUME_EARLY, + SUSPEND_RESUME +}; + +void dpm_save_failed_dev(const char *name); +void dpm_save_failed_step(enum suspend_stat_step step); + #endif /* _LINUX_SUSPEND_H */ diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index ecde0312dd52..ea23097e351f 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -120,6 +120,8 @@ struct io_tlb_pool { * debugfs. * @used_hiwater: The high water mark for total_used. Used only for reporting * in debugfs. + * @transient_nslabs: The total number of slots in all transient pools that + * are currently used across all areas. */ struct io_tlb_mem { struct io_tlb_pool defpool; @@ -137,6 +139,7 @@ struct io_tlb_mem { #ifdef CONFIG_DEBUG_FS atomic_long_t total_used; atomic_long_t used_hiwater; + atomic_long_t transient_nslabs; #endif }; diff --git a/include/linux/thermal.h b/include/linux/thermal.h index b7a3deb372fd..c33f50177f51 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -64,15 +64,23 @@ enum thermal_notify_event { * @threshold: trip crossing notification threshold miliCelsius * @type: trip point type * @priv: pointer to driver data associated with this trip + * @flags: flags representing binary properties of the trip */ struct thermal_trip { int temperature; int hysteresis; int threshold; enum thermal_trip_type type; + u8 flags; void *priv; }; +#define THERMAL_TRIP_FLAG_RW_TEMP BIT(0) +#define THERMAL_TRIP_FLAG_RW_HYST BIT(1) + +#define THERMAL_TRIP_FLAG_RW (THERMAL_TRIP_FLAG_RW_TEMP | \ + THERMAL_TRIP_FLAG_RW_HYST) + struct thermal_zone_device_ops { int (*bind) (struct thermal_zone_device *, struct thermal_cooling_device *); @@ -83,7 +91,6 @@ struct thermal_zone_device_ops { int (*change_mode) (struct thermal_zone_device *, enum thermal_device_mode); int (*set_trip_temp) (struct thermal_zone_device *, int, int); - int (*set_trip_hyst) (struct thermal_zone_device *, int, int); int (*get_crit_temp) (struct thermal_zone_device *, int *); int (*set_emul_temp) (struct thermal_zone_device *, int); int (*get_trend) (struct thermal_zone_device *, @@ -130,7 +137,6 @@ struct thermal_cooling_device { * @trip_hyst_attrs: attributes for trip points for sysfs: trip hysteresis * @mode: current mode of this thermal zone * @devdata: private pointer for device private data - * @trips: an array of struct thermal_trip * @num_trips: number of trip points the thermal zone supports * @passive_delay_jiffies: number of jiffies to wait between polls when * performing passive cooling. @@ -160,6 +166,7 @@ struct thermal_cooling_device { * @poll_queue: delayed work for polling * @notify_event: Last notification event * @suspended: thermal zone suspend indicator + * @trips: array of struct thermal_trip objects */ struct thermal_zone_device { int id; @@ -172,7 +179,6 @@ struct thermal_zone_device { struct thermal_attr *trip_hyst_attrs; enum thermal_device_mode mode; void *devdata; - struct thermal_trip *trips; int num_trips; unsigned long passive_delay_jiffies; unsigned long polling_delay_jiffies; @@ -183,7 +189,7 @@ struct thermal_zone_device { int prev_low_trip; int prev_high_trip; atomic_t need_update; - struct thermal_zone_device_ops *ops; + struct thermal_zone_device_ops ops; struct thermal_zone_params *tzp; struct thermal_governor *governor; void *governor_data; @@ -193,10 +199,11 @@ struct thermal_zone_device { struct list_head node; struct delayed_work poll_queue; enum thermal_notify_event notify_event; + bool suspended; #ifdef CONFIG_THERMAL_DEBUGFS struct thermal_debugfs *debugfs; #endif - bool suspended; + struct thermal_trip trips[] __counted_by(num_trips); }; /** @@ -214,7 +221,7 @@ struct thermal_zone_device { * @governor_list: node in thermal_governor_list (in thermal_core.c) */ struct thermal_governor { - char name[THERMAL_NAME_LENGTH]; + const char *name; int (*bind_to_tz)(struct thermal_zone_device *tz); void (*unbind_from_tz)(struct thermal_zone_device *tz); int (*throttle)(struct thermal_zone_device *tz, @@ -226,7 +233,7 @@ struct thermal_governor { /* Structure to define Thermal Zone parameters */ struct thermal_zone_params { - char governor_name[THERMAL_NAME_LENGTH]; + const char *governor_name; /* * a boolean to indicate if the thermal to hwmon sysfs interface @@ -315,17 +322,16 @@ int thermal_zone_get_crit_temp(struct thermal_zone_device *tz, int *temp); #ifdef CONFIG_THERMAL struct thermal_zone_device *thermal_zone_device_register_with_trips( const char *type, - struct thermal_trip *trips, - int num_trips, int mask, - void *devdata, - struct thermal_zone_device_ops *ops, + const struct thermal_trip *trips, + int num_trips, void *devdata, + const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp, int passive_delay, int polling_delay); struct thermal_zone_device *thermal_tripless_zone_device_register( const char *type, void *devdata, - struct thermal_zone_device_ops *ops, + const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp); void thermal_zone_device_unregister(struct thermal_zone_device *tz); @@ -375,10 +381,9 @@ void thermal_zone_device_critical(struct thermal_zone_device *tz); #else static inline struct thermal_zone_device *thermal_zone_device_register_with_trips( const char *type, - struct thermal_trip *trips, - int num_trips, int mask, - void *devdata, - struct thermal_zone_device_ops *ops, + const struct thermal_trip *trips, + int num_trips, void *devdata, + const struct thermal_zone_device_ops *ops, const struct thermal_zone_params *tzp, int passive_delay, int polling_delay) { return ERR_PTR(-ENODEV); } diff --git a/include/media/v4l2-common.h b/include/media/v4l2-common.h index acf5be24a5ca..739b0f0fc1a0 100644 --- a/include/media/v4l2-common.h +++ b/include/media/v4l2-common.h @@ -278,13 +278,13 @@ static inline void v4l2_i2c_subdev_unregister(struct v4l2_subdev *sd) * * * @v4l2_dev: pointer to &struct v4l2_device. - * @master: pointer to struct spi_master. + * @ctlr: pointer to struct spi_controller. * @info: pointer to struct spi_board_info. * * returns a &struct v4l2_subdev pointer. */ struct v4l2_subdev *v4l2_spi_new_subdev(struct v4l2_device *v4l2_dev, - struct spi_master *master, struct spi_board_info *info); + struct spi_controller *ctlr, struct spi_board_info *info); /** * v4l2_spi_subdev_init - Initialize a v4l2_subdev with data from an @@ -308,7 +308,7 @@ void v4l2_spi_subdev_unregister(struct v4l2_subdev *sd); static inline struct v4l2_subdev * v4l2_spi_new_subdev(struct v4l2_device *v4l2_dev, - struct spi_master *master, struct spi_board_info *info) + struct spi_controller *ctlr, struct spi_board_info *info) { return NULL; } diff --git a/include/trace/events/rpm.h b/include/trace/events/rpm.h index 3c716214dab1..bd120e23ce12 100644 --- a/include/trace/events/rpm.h +++ b/include/trace/events/rpm.h @@ -101,6 +101,48 @@ TRACE_EVENT(rpm_return_int, __entry->ret) ); +#define RPM_STATUS_STRINGS \ + EM(RPM_INVALID, "RPM_INVALID") \ + EM(RPM_ACTIVE, "RPM_ACTIVE") \ + EM(RPM_RESUMING, "RPM_RESUMING") \ + EM(RPM_SUSPENDED, "RPM_SUSPENDED") \ + EMe(RPM_SUSPENDING, "RPM_SUSPENDING") + +/* Enums require being exported to userspace, for user tool parsing. */ +#undef EM +#undef EMe +#define EM(a, b) TRACE_DEFINE_ENUM(a); +#define EMe(a, b) TRACE_DEFINE_ENUM(a); + +RPM_STATUS_STRINGS + +/* + * Now redefine the EM() and EMe() macros to map the enums to the strings that + * will be printed in the output. + */ +#undef EM +#undef EMe +#define EM(a, b) { a, b }, +#define EMe(a, b) { a, b } + +TRACE_EVENT(rpm_status, + TP_PROTO(struct device *dev, enum rpm_status status), + TP_ARGS(dev, status), + + TP_STRUCT__entry( + __string(name, dev_name(dev)) + __field(int, status) + ), + + TP_fast_assign( + __assign_str(name, dev_name(dev)); + __entry->status = status; + ), + + TP_printk("%s status=%s", __get_str(name), + __print_symbolic(__entry->status, RPM_STATUS_STRINGS)) +); + #endif /* _TRACE_RUNTIME_POWER_H */ /* This part must be outside protection */ diff --git a/include/uapi/linux/gpio.h b/include/uapi/linux/gpio.h index cb9966d49a16..f7cb8ae87df7 100644 --- a/include/uapi/linux/gpio.h +++ b/include/uapi/linux/gpio.h @@ -67,7 +67,7 @@ struct gpiochip_info { * @GPIO_V2_LINE_FLAG_BIAS_DISABLED: line has bias disabled * @GPIO_V2_LINE_FLAG_EVENT_CLOCK_REALTIME: line events contain REALTIME timestamps * @GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE: line events contain timestamps from - * hardware timestamp engine + * the hardware timestamping engine (HTE) subsystem */ enum gpio_v2_line_flag { GPIO_V2_LINE_FLAG_USED = _BITULL(0), @@ -88,10 +88,10 @@ enum gpio_v2_line_flag { /** * struct gpio_v2_line_values - Values of GPIO lines * @bits: a bitmap containing the value of the lines, set to 1 for active - * and 0 for inactive. + * and 0 for inactive * @mask: a bitmap identifying the lines to get or set, with each bit * number corresponding to the index into &struct - * gpio_v2_line_request.offsets. + * gpio_v2_line_request.offsets */ struct gpio_v2_line_values { __aligned_u64 bits; @@ -123,7 +123,7 @@ enum gpio_v2_line_attr_id { * @values: if id is %GPIO_V2_LINE_ATTR_ID_OUTPUT_VALUES, a bitmap * containing the values to which the lines will be set, with each bit * number corresponding to the index into &struct - * gpio_v2_line_request.offsets. + * gpio_v2_line_request.offsets * @debounce_period_us: if id is %GPIO_V2_LINE_ATTR_ID_DEBOUNCE, the * desired debounce period, in microseconds */ @@ -143,7 +143,7 @@ struct gpio_v2_line_attribute { * @attr: the configurable attribute * @mask: a bitmap identifying the lines to which the attribute applies, * with each bit number corresponding to the index into &struct - * gpio_v2_line_request.offsets. + * gpio_v2_line_request.offsets */ struct gpio_v2_line_config_attribute { struct gpio_v2_line_attribute attr; @@ -178,7 +178,7 @@ struct gpio_v2_line_config { * associated GPIO chip * @consumer: a desired consumer label for the selected GPIO lines such as * "my-bitbanged-relay" - * @config: requested configuration for the lines. + * @config: requested configuration for the lines * @num_lines: number of lines requested in this request, i.e. the number * of valid fields in the %GPIO_V2_LINES_MAX sized arrays, set to 1 to * request a single line @@ -189,9 +189,8 @@ struct gpio_v2_line_config { * buffer. If this field is zero then the buffer size defaults to a minimum * of @num_lines * 16. * @padding: reserved for future use and must be zero filled - * @fd: if successful this field will contain a valid anonymous file handle - * after a %GPIO_GET_LINE_IOCTL operation, zero or negative value means - * error + * @fd: after a successful %GPIO_V2_GET_LINE_IOCTL operation, contains + * a valid anonymous file descriptor representing the request */ struct gpio_v2_line_request { __u32 offsets[GPIO_V2_LINES_MAX]; @@ -217,7 +216,7 @@ struct gpio_v2_line_request { * @num_attrs: the number of attributes in @attrs * @flags: flags for this GPIO line, with values from &enum * gpio_v2_line_flag, such as %GPIO_V2_LINE_FLAG_ACTIVE_LOW, - * %GPIO_V2_LINE_FLAG_OUTPUT etc, added together. + * %GPIO_V2_LINE_FLAG_OUTPUT etc, added together * @attrs: the configuration attributes associated with the line * @padding: reserved for future use */ @@ -274,7 +273,7 @@ enum gpio_v2_line_event_id { /** * struct gpio_v2_line_event - The actual event being pushed to userspace - * @timestamp_ns: best estimate of time of event occurrence, in nanoseconds. + * @timestamp_ns: best estimate of time of event occurrence, in nanoseconds * @id: event identifier with value from &enum gpio_v2_line_event_id * @offset: the offset of the line that triggered the event * @seqno: the sequence number for this event in the sequence of events for @@ -289,6 +288,10 @@ enum gpio_v2_line_event_id { * * If the %GPIO_V2_LINE_FLAG_EVENT_CLOCK_REALTIME flag is set then the * @timestamp_ns is read from %CLOCK_REALTIME. + * + * If the %GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE flag is set then the + * @timestamp_ns is provided by the hardware timestamping engine (HTE) + * subsystem. */ struct gpio_v2_line_event { __aligned_u64 timestamp_ns; @@ -330,7 +333,7 @@ struct gpio_v2_line_event { * also be empty if the consumer doesn't set this up * * Note: This struct is part of ABI v1 and is deprecated. - * Use &struct gpio_v2_line_info instead. + * Use ABI v2 and &struct gpio_v2_line_info instead. */ struct gpioline_info { __u32 line_offset; @@ -365,7 +368,7 @@ enum { * at the end of the structure on 64-bit architectures. * * Note: This struct is part of ABI v1 and is deprecated. - * Use &struct gpio_v2_line_info_changed instead. + * Use ABI v2 and &struct gpio_v2_line_info_changed instead. */ struct gpioline_info_changed { struct gpioline_info info; @@ -396,18 +399,17 @@ struct gpioline_info_changed { * a batch of input or output lines, but they must all have the same * characteristics, i.e. all inputs or all outputs, all active low etc * @default_values: if the %GPIOHANDLE_REQUEST_OUTPUT is set for a requested - * line, this specifies the default output value, should be 0 (low) or - * 1 (high), anything else than 0 or 1 will be interpreted as 1 (high) + * line, this specifies the default output value, should be 0 (inactive) or + * 1 (active). Anything other than 0 or 1 will be interpreted as active. * @consumer_label: a desired consumer label for the selected GPIO line(s) * such as "my-bitbanged-relay" * @lines: number of lines requested in this request, i.e. the number of * valid fields in the above arrays, set to 1 to request a single line - * @fd: if successful this field will contain a valid anonymous file handle - * after a %GPIO_GET_LINEHANDLE_IOCTL operation, zero or negative value - * means error + * @fd: after a successful %GPIO_GET_LINEHANDLE_IOCTL operation, contains + * a valid anonymous file descriptor representing the request * * Note: This struct is part of ABI v1 and is deprecated. - * Use &struct gpio_v2_line_request instead. + * Use ABI v2 and &struct gpio_v2_line_request instead. */ struct gpiohandle_request { __u32 lineoffsets[GPIOHANDLES_MAX]; @@ -424,12 +426,12 @@ struct gpiohandle_request { * %GPIOHANDLE_REQUEST_OUTPUT, %GPIOHANDLE_REQUEST_ACTIVE_LOW etc, added * together * @default_values: if the %GPIOHANDLE_REQUEST_OUTPUT is set in flags, - * this specifies the default output value, should be 0 (low) or - * 1 (high), anything else than 0 or 1 will be interpreted as 1 (high) + * this specifies the default output value, should be 0 (inactive) or + * 1 (active). Anything other than 0 or 1 will be interpreted as active. * @padding: reserved for future use and should be zero filled * * Note: This struct is part of ABI v1 and is deprecated. - * Use &struct gpio_v2_line_config instead. + * Use ABI v2 and &struct gpio_v2_line_config instead. */ struct gpiohandle_config { __u32 flags; @@ -441,10 +443,11 @@ struct gpiohandle_config { * struct gpiohandle_data - Information of values on a GPIO handle * @values: when getting the state of lines this contains the current * state of a line, when setting the state of lines these should contain - * the desired target state + * the desired target state. States are 0 (inactive) or 1 (active). + * When setting, anything other than 0 or 1 will be interpreted as active. * * Note: This struct is part of ABI v1 and is deprecated. - * Use &struct gpio_v2_line_values instead. + * Use ABI v2 and &struct gpio_v2_line_values instead. */ struct gpiohandle_data { __u8 values[GPIOHANDLES_MAX]; @@ -465,12 +468,11 @@ struct gpiohandle_data { * %GPIOEVENT_REQUEST_RISING_EDGE or %GPIOEVENT_REQUEST_FALLING_EDGE * @consumer_label: a desired consumer label for the selected GPIO line(s) * such as "my-listener" - * @fd: if successful this field will contain a valid anonymous file handle - * after a %GPIO_GET_LINEEVENT_IOCTL operation, zero or negative value - * means error + * @fd: after a successful %GPIO_GET_LINEEVENT_IOCTL operation, contains a + * valid anonymous file descriptor representing the request * * Note: This struct is part of ABI v1 and is deprecated. - * Use &struct gpio_v2_line_request instead. + * Use ABI v2 and &struct gpio_v2_line_request instead. */ struct gpioevent_request { __u32 lineoffset; @@ -489,10 +491,11 @@ struct gpioevent_request { /** * struct gpioevent_data - The actual event being pushed to userspace * @timestamp: best estimate of time of event occurrence, in nanoseconds - * @id: event identifier + * @id: event identifier, one of %GPIOEVENT_EVENT_RISING_EDGE or + * %GPIOEVENT_EVENT_FALLING_EDGE * * Note: This struct is part of ABI v1 and is deprecated. - * Use &struct gpio_v2_line_event instead. + * Use ABI v2 and &struct gpio_v2_line_event instead. */ struct gpioevent_data { __u64 timestamp; diff --git a/include/uapi/linux/iommu.h b/include/uapi/linux/iommu.h deleted file mode 100644 index 65d8b0234f69..000000000000 --- a/include/uapi/linux/iommu.h +++ /dev/null @@ -1,161 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * IOMMU user API definitions - */ - -#ifndef _UAPI_IOMMU_H -#define _UAPI_IOMMU_H - -#include - -#define IOMMU_FAULT_PERM_READ (1 << 0) /* read */ -#define IOMMU_FAULT_PERM_WRITE (1 << 1) /* write */ -#define IOMMU_FAULT_PERM_EXEC (1 << 2) /* exec */ -#define IOMMU_FAULT_PERM_PRIV (1 << 3) /* privileged */ - -/* Generic fault types, can be expanded IRQ remapping fault */ -enum iommu_fault_type { - IOMMU_FAULT_DMA_UNRECOV = 1, /* unrecoverable fault */ - IOMMU_FAULT_PAGE_REQ, /* page request fault */ -}; - -enum iommu_fault_reason { - IOMMU_FAULT_REASON_UNKNOWN = 0, - - /* Could not access the PASID table (fetch caused external abort) */ - IOMMU_FAULT_REASON_PASID_FETCH, - - /* PASID entry is invalid or has configuration errors */ - IOMMU_FAULT_REASON_BAD_PASID_ENTRY, - - /* - * PASID is out of range (e.g. exceeds the maximum PASID - * supported by the IOMMU) or disabled. - */ - IOMMU_FAULT_REASON_PASID_INVALID, - - /* - * An external abort occurred fetching (or updating) a translation - * table descriptor - */ - IOMMU_FAULT_REASON_WALK_EABT, - - /* - * Could not access the page table entry (Bad address), - * actual translation fault - */ - IOMMU_FAULT_REASON_PTE_FETCH, - - /* Protection flag check failed */ - IOMMU_FAULT_REASON_PERMISSION, - - /* access flag check failed */ - IOMMU_FAULT_REASON_ACCESS, - - /* Output address of a translation stage caused Address Size fault */ - IOMMU_FAULT_REASON_OOR_ADDRESS, -}; - -/** - * struct iommu_fault_unrecoverable - Unrecoverable fault data - * @reason: reason of the fault, from &enum iommu_fault_reason - * @flags: parameters of this fault (IOMMU_FAULT_UNRECOV_* values) - * @pasid: Process Address Space ID - * @perm: requested permission access using by the incoming transaction - * (IOMMU_FAULT_PERM_* values) - * @addr: offending page address - * @fetch_addr: address that caused a fetch abort, if any - */ -struct iommu_fault_unrecoverable { - __u32 reason; -#define IOMMU_FAULT_UNRECOV_PASID_VALID (1 << 0) -#define IOMMU_FAULT_UNRECOV_ADDR_VALID (1 << 1) -#define IOMMU_FAULT_UNRECOV_FETCH_ADDR_VALID (1 << 2) - __u32 flags; - __u32 pasid; - __u32 perm; - __u64 addr; - __u64 fetch_addr; -}; - -/** - * struct iommu_fault_page_request - Page Request data - * @flags: encodes whether the corresponding fields are valid and whether this - * is the last page in group (IOMMU_FAULT_PAGE_REQUEST_* values). - * When IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID is set, the page response - * must have the same PASID value as the page request. When it is clear, - * the page response should not have a PASID. - * @pasid: Process Address Space ID - * @grpid: Page Request Group Index - * @perm: requested page permissions (IOMMU_FAULT_PERM_* values) - * @addr: page address - * @private_data: device-specific private information - */ -struct iommu_fault_page_request { -#define IOMMU_FAULT_PAGE_REQUEST_PASID_VALID (1 << 0) -#define IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE (1 << 1) -#define IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA (1 << 2) -#define IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID (1 << 3) - __u32 flags; - __u32 pasid; - __u32 grpid; - __u32 perm; - __u64 addr; - __u64 private_data[2]; -}; - -/** - * struct iommu_fault - Generic fault data - * @type: fault type from &enum iommu_fault_type - * @padding: reserved for future use (should be zero) - * @event: fault event, when @type is %IOMMU_FAULT_DMA_UNRECOV - * @prm: Page Request message, when @type is %IOMMU_FAULT_PAGE_REQ - * @padding2: sets the fault size to allow for future extensions - */ -struct iommu_fault { - __u32 type; - __u32 padding; - union { - struct iommu_fault_unrecoverable event; - struct iommu_fault_page_request prm; - __u8 padding2[56]; - }; -}; - -/** - * enum iommu_page_response_code - Return status of fault handlers - * @IOMMU_PAGE_RESP_SUCCESS: Fault has been handled and the page tables - * populated, retry the access. This is "Success" in PCI PRI. - * @IOMMU_PAGE_RESP_FAILURE: General error. Drop all subsequent faults from - * this device if possible. This is "Response Failure" in PCI PRI. - * @IOMMU_PAGE_RESP_INVALID: Could not handle this fault, don't retry the - * access. This is "Invalid Request" in PCI PRI. - */ -enum iommu_page_response_code { - IOMMU_PAGE_RESP_SUCCESS = 0, - IOMMU_PAGE_RESP_INVALID, - IOMMU_PAGE_RESP_FAILURE, -}; - -/** - * struct iommu_page_response - Generic page response information - * @argsz: User filled size of this data - * @version: API version of this structure - * @flags: encodes whether the corresponding fields are valid - * (IOMMU_FAULT_PAGE_RESPONSE_* values) - * @pasid: Process Address Space ID - * @grpid: Page Request Group Index - * @code: response code from &enum iommu_page_response_code - */ -struct iommu_page_response { - __u32 argsz; -#define IOMMU_PAGE_RESP_VERSION_1 1 - __u32 version; -#define IOMMU_PAGE_RESP_PASID_VALID (1 << 0) - __u32 flags; - __u32 pasid; - __u32 grpid; - __u32 code; -}; - -#endif /* _UAPI_IOMMU_H */ diff --git a/include/uapi/linux/lsm.h b/include/uapi/linux/lsm.h index f8aef9ade549..33d8c9f4aa6b 100644 --- a/include/uapi/linux/lsm.h +++ b/include/uapi/linux/lsm.h @@ -62,6 +62,8 @@ struct lsm_ctx { #define LSM_ID_LOCKDOWN 108 #define LSM_ID_BPF 109 #define LSM_ID_LANDLOCK 110 +#define LSM_ID_IMA 111 +#define LSM_ID_EVM 112 /* * LSM_ATTR_XXX definitions identify different LSM attributes diff --git a/kernel/cred.c b/kernel/cred.c index a9895450a547..e9edcd176d98 100644 --- a/kernel/cred.c +++ b/kernel/cred.c @@ -612,8 +612,8 @@ int set_cred_ucounts(struct cred *new) void __init cred_init(void) { /* allocate a slab in which we can store credentials */ - cred_jar = kmem_cache_create("cred_jar", sizeof(struct cred), 0, - SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); + cred_jar = KMEM_CACHE(cred, + SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT); } /** diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 98b2e192fd69..4d543b1e9d57 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -286,7 +286,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, } else { ret = page_address(page); if (dma_set_decrypted(dev, ret, size)) - goto out_free_pages; + goto out_leak_pages; } memset(ret, 0, size); @@ -307,6 +307,8 @@ out_encrypt_pages: out_free_pages: __dma_direct_free_pages(dev, page, size); return NULL; +out_leak_pages: + return NULL; } void dma_direct_free(struct device *dev, size_t size, @@ -367,12 +369,11 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size, ret = page_address(page); if (dma_set_decrypted(dev, ret, size)) - goto out_free_pages; + goto out_leak_pages; memset(ret, 0, size); *dma_handle = phys_to_dma_direct(dev, page_to_phys(page)); return page; -out_free_pages: - __dma_direct_free_pages(dev, page, size); +out_leak_pages: return NULL; } diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index b079a9a8e087..77974cea3e69 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -956,6 +956,28 @@ static void dec_used(struct io_tlb_mem *mem, unsigned int nslots) } #endif /* CONFIG_DEBUG_FS */ +#ifdef CONFIG_SWIOTLB_DYNAMIC +#ifdef CONFIG_DEBUG_FS +static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots) +{ + atomic_long_add(nslots, &mem->transient_nslabs); +} + +static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots) +{ + atomic_long_sub(nslots, &mem->transient_nslabs); +} + +#else /* !CONFIG_DEBUG_FS */ +static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots) +{ +} +static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots) +{ +} +#endif /* CONFIG_DEBUG_FS */ +#endif /* CONFIG_SWIOTLB_DYNAMIC */ + /** * swiotlb_search_pool_area() - search one memory area in one pool * @dev: Device which maps the buffer. @@ -1170,6 +1192,7 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr, spin_lock_irqsave(&dev->dma_io_tlb_lock, flags); list_add_rcu(&pool->node, &dev->dma_io_tlb_pools); spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags); + inc_transient_used(mem, pool->nslabs); found: WRITE_ONCE(dev->dma_uses_io_tlb, true); @@ -1415,6 +1438,7 @@ static bool swiotlb_del_transient(struct device *dev, phys_addr_t tlb_addr) dec_used(dev->dma_io_tlb_mem, pool->nslabs); swiotlb_del_pool(dev, pool); + dec_transient_used(dev->dma_io_tlb_mem, pool->nslabs); return true; } @@ -1557,6 +1581,23 @@ phys_addr_t default_swiotlb_limit(void) } #ifdef CONFIG_DEBUG_FS +#ifdef CONFIG_SWIOTLB_DYNAMIC +static unsigned long mem_transient_used(struct io_tlb_mem *mem) +{ + return atomic_long_read(&mem->transient_nslabs); +} + +static int io_tlb_transient_used_get(void *data, u64 *val) +{ + struct io_tlb_mem *mem = data; + + *val = mem_transient_used(mem); + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_transient_used, io_tlb_transient_used_get, + NULL, "%llu\n"); +#endif /* CONFIG_SWIOTLB_DYNAMIC */ static int io_tlb_used_get(void *data, u64 *val) { @@ -1605,6 +1646,11 @@ static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem, &fops_io_tlb_used); debugfs_create_file("io_tlb_used_hiwater", 0600, mem->debugfs, mem, &fops_io_tlb_hiwater); +#ifdef CONFIG_SWIOTLB_DYNAMIC + atomic_long_set(&mem->transient_nslabs, 0); + debugfs_create_file("io_tlb_transient_nslabs", 0400, mem->debugfs, + mem, &fops_io_tlb_transient_used); +#endif } static int __init swiotlb_create_default_debugfs(void) diff --git a/kernel/panic.c b/kernel/panic.c index 2807639aab51..f22d8f33ea14 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -446,6 +446,14 @@ void panic(const char *fmt, ...) /* Do not scroll important messages printed above */ suppress_printk = 1; + + /* + * The final messages may not have been printed if in a context that + * defers printing (such as NMI) and irq_work is not available. + * Explicitly flush the kernel log buffer one last time. + */ + console_flush_on_panic(CONSOLE_FLUSH_PENDING); + local_irq_enable(); for (i = 0; ; i += PANIC_TIMER_STEP) { touch_softlockup_watchdog(); diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig index 4b31629c5be4..afce8130d8b9 100644 --- a/kernel/power/Kconfig +++ b/kernel/power/Kconfig @@ -39,9 +39,9 @@ config HIBERNATION bool "Hibernation (aka 'suspend to disk')" depends on SWAP && ARCH_HIBERNATION_POSSIBLE select HIBERNATE_CALLBACKS - select LZO_COMPRESS - select LZO_DECOMPRESS select CRC32 + select CRYPTO + select CRYPTO_LZO help Enable the suspend to disk (STD) functionality, which is usually called "hibernation" in user interfaces. STD checkpoints the @@ -92,6 +92,28 @@ config HIBERNATION_SNAPSHOT_DEV If in doubt, say Y. +choice + prompt "Default compressor" + default HIBERNATION_COMP_LZO + depends on HIBERNATION + +config HIBERNATION_COMP_LZO + bool "lzo" + depends on CRYPTO_LZO + +config HIBERNATION_COMP_LZ4 + bool "lz4" + depends on CRYPTO_LZ4 + +endchoice + +config HIBERNATION_DEF_COMP + string + default "lzo" if HIBERNATION_COMP_LZO + default "lz4" if HIBERNATION_COMP_LZ4 + help + Default compressor to be used for hibernation. + config PM_STD_PARTITION string "Default resume partition" depends on HIBERNATION diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c index 7b44f5b89fa1..b686ac0345bd 100644 --- a/kernel/power/energy_model.c +++ b/kernel/power/energy_model.c @@ -23,6 +23,12 @@ */ static DEFINE_MUTEX(em_pd_mutex); +static void em_cpufreq_update_efficiencies(struct device *dev, + struct em_perf_state *table); +static void em_check_capacity_update(void); +static void em_update_workfn(struct work_struct *work); +static DECLARE_DELAYED_WORK(em_update_work, em_update_workfn); + static bool _is_cpu_device(struct device *dev) { return (dev->bus == &cpu_subsys); @@ -31,19 +37,65 @@ static bool _is_cpu_device(struct device *dev) #ifdef CONFIG_DEBUG_FS static struct dentry *rootdir; -static void em_debug_create_ps(struct em_perf_state *ps, struct dentry *pd) +struct em_dbg_info { + struct em_perf_domain *pd; + int ps_id; +}; + +#define DEFINE_EM_DBG_SHOW(name, fname) \ +static int em_debug_##fname##_show(struct seq_file *s, void *unused) \ +{ \ + struct em_dbg_info *em_dbg = s->private; \ + struct em_perf_state *table; \ + unsigned long val; \ + \ + rcu_read_lock(); \ + table = em_perf_state_from_pd(em_dbg->pd); \ + val = table[em_dbg->ps_id].name; \ + rcu_read_unlock(); \ + \ + seq_printf(s, "%lu\n", val); \ + return 0; \ +} \ +DEFINE_SHOW_ATTRIBUTE(em_debug_##fname) + +DEFINE_EM_DBG_SHOW(frequency, frequency); +DEFINE_EM_DBG_SHOW(power, power); +DEFINE_EM_DBG_SHOW(cost, cost); +DEFINE_EM_DBG_SHOW(performance, performance); +DEFINE_EM_DBG_SHOW(flags, inefficiency); + +static void em_debug_create_ps(struct em_perf_domain *em_pd, + struct em_dbg_info *em_dbg, int i, + struct dentry *pd) { + struct em_perf_state *table; + unsigned long freq; struct dentry *d; char name[24]; - snprintf(name, sizeof(name), "ps:%lu", ps->frequency); + em_dbg[i].pd = em_pd; + em_dbg[i].ps_id = i; + + rcu_read_lock(); + table = em_perf_state_from_pd(em_pd); + freq = table[i].frequency; + rcu_read_unlock(); + + snprintf(name, sizeof(name), "ps:%lu", freq); /* Create per-ps directory */ d = debugfs_create_dir(name, pd); - debugfs_create_ulong("frequency", 0444, d, &ps->frequency); - debugfs_create_ulong("power", 0444, d, &ps->power); - debugfs_create_ulong("cost", 0444, d, &ps->cost); - debugfs_create_ulong("inefficient", 0444, d, &ps->flags); + debugfs_create_file("frequency", 0444, d, &em_dbg[i], + &em_debug_frequency_fops); + debugfs_create_file("power", 0444, d, &em_dbg[i], + &em_debug_power_fops); + debugfs_create_file("cost", 0444, d, &em_dbg[i], + &em_debug_cost_fops); + debugfs_create_file("performance", 0444, d, &em_dbg[i], + &em_debug_performance_fops); + debugfs_create_file("inefficient", 0444, d, &em_dbg[i], + &em_debug_inefficiency_fops); } static int em_debug_cpus_show(struct seq_file *s, void *unused) @@ -66,6 +118,7 @@ DEFINE_SHOW_ATTRIBUTE(em_debug_flags); static void em_debug_create_pd(struct device *dev) { + struct em_dbg_info *em_dbg; struct dentry *d; int i; @@ -79,9 +132,14 @@ static void em_debug_create_pd(struct device *dev) debugfs_create_file("flags", 0444, d, dev->em_pd, &em_debug_flags_fops); + em_dbg = devm_kcalloc(dev, dev->em_pd->nr_perf_states, + sizeof(*em_dbg), GFP_KERNEL); + if (!em_dbg) + return; + /* Create a sub-directory for each performance state */ for (i = 0; i < dev->em_pd->nr_perf_states; i++) - em_debug_create_ps(&dev->em_pd->table[i], d); + em_debug_create_ps(dev->em_pd, em_dbg, i, d); } @@ -103,72 +161,105 @@ static void em_debug_create_pd(struct device *dev) {} static void em_debug_remove_pd(struct device *dev) {} #endif -static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, - int nr_states, struct em_data_callback *cb, - unsigned long flags) +static void em_destroy_table_rcu(struct rcu_head *rp) { - unsigned long power, freq, prev_freq = 0, prev_cost = ULONG_MAX; - struct em_perf_state *table; - int i, ret; - u64 fmax; + struct em_perf_table __rcu *table; - table = kcalloc(nr_states, sizeof(*table), GFP_KERNEL); + table = container_of(rp, struct em_perf_table, rcu); + kfree(table); +} + +static void em_release_table_kref(struct kref *kref) +{ + struct em_perf_table __rcu *table; + + /* It was the last owner of this table so we can free */ + table = container_of(kref, struct em_perf_table, kref); + + call_rcu(&table->rcu, em_destroy_table_rcu); +} + +/** + * em_table_free() - Handles safe free of the EM table when needed + * @table : EM table which is going to be freed + * + * No return values. + */ +void em_table_free(struct em_perf_table __rcu *table) +{ + kref_put(&table->kref, em_release_table_kref); +} + +/** + * em_table_alloc() - Allocate a new EM table + * @pd : EM performance domain for which this must be done + * + * Allocate a new EM table and initialize its kref to indicate that it + * has a user. + * Returns allocated table or NULL. + */ +struct em_perf_table __rcu *em_table_alloc(struct em_perf_domain *pd) +{ + struct em_perf_table __rcu *table; + int table_size; + + table_size = sizeof(struct em_perf_state) * pd->nr_perf_states; + + table = kzalloc(sizeof(*table) + table_size, GFP_KERNEL); if (!table) - return -ENOMEM; + return NULL; - /* Build the list of performance states for this performance domain */ - for (i = 0, freq = 0; i < nr_states; i++, freq++) { - /* - * active_power() is a driver callback which ceils 'freq' to - * lowest performance state of 'dev' above 'freq' and updates - * 'power' and 'freq' accordingly. - */ - ret = cb->active_power(dev, &power, &freq); - if (ret) { - dev_err(dev, "EM: invalid perf. state: %d\n", - ret); - goto free_ps_table; - } + kref_init(&table->kref); - /* - * We expect the driver callback to increase the frequency for - * higher performance states. - */ - if (freq <= prev_freq) { - dev_err(dev, "EM: non-increasing freq: %lu\n", - freq); - goto free_ps_table; - } + return table; +} - /* - * The power returned by active_state() is expected to be - * positive and be in range. - */ - if (!power || power > EM_MAX_POWER) { - dev_err(dev, "EM: invalid power: %lu\n", - power); - goto free_ps_table; - } +static void em_init_performance(struct device *dev, struct em_perf_domain *pd, + struct em_perf_state *table, int nr_states) +{ + u64 fmax, max_cap; + int i, cpu; - table[i].power = power; - table[i].frequency = prev_freq = freq; - } + /* This is needed only for CPUs and EAS skip other devices */ + if (!_is_cpu_device(dev)) + return; + + cpu = cpumask_first(em_span_cpus(pd)); + + /* + * Calculate the performance value for each frequency with + * linear relationship. The final CPU capacity might not be ready at + * boot time, but the EM will be updated a bit later with correct one. + */ + fmax = (u64) table[nr_states - 1].frequency; + max_cap = (u64) arch_scale_cpu_capacity(cpu); + for (i = 0; i < nr_states; i++) + table[i].performance = div64_u64(max_cap * table[i].frequency, + fmax); +} + +static int em_compute_costs(struct device *dev, struct em_perf_state *table, + struct em_data_callback *cb, int nr_states, + unsigned long flags) +{ + unsigned long prev_cost = ULONG_MAX; + int i, ret; /* Compute the cost of each performance state. */ - fmax = (u64) table[nr_states - 1].frequency; for (i = nr_states - 1; i >= 0; i--) { unsigned long power_res, cost; - if (flags & EM_PERF_DOMAIN_ARTIFICIAL) { + if ((flags & EM_PERF_DOMAIN_ARTIFICIAL) && cb->get_cost) { ret = cb->get_cost(dev, table[i].frequency, &cost); if (ret || !cost || cost > EM_MAX_POWER) { dev_err(dev, "EM: invalid cost %lu %d\n", cost, ret); - goto free_ps_table; + return -EINVAL; } } else { - power_res = table[i].power; - cost = div64_u64(fmax * power_res, table[i].frequency); + /* increase resolution of 'cost' precision */ + power_res = table[i].power * 10; + cost = power_res / table[i].performance; } table[i].cost = cost; @@ -182,20 +273,133 @@ static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, } } - pd->table = table; - pd->nr_perf_states = nr_states; + return 0; +} + +/** + * em_dev_compute_costs() - Calculate cost values for new runtime EM table + * @dev : Device for which the EM table is to be updated + * @table : The new EM table that is going to get the costs calculated + * @nr_states : Number of performance states + * + * Calculate the em_perf_state::cost values for new runtime EM table. The + * values are used for EAS during task placement. It also calculates and sets + * the efficiency flag for each performance state. When the function finish + * successfully the EM table is ready to be updated and used by EAS. + * + * Return 0 on success or a proper error in case of failure. + */ +int em_dev_compute_costs(struct device *dev, struct em_perf_state *table, + int nr_states) +{ + return em_compute_costs(dev, table, NULL, nr_states, 0); +} + +/** + * em_dev_update_perf_domain() - Update runtime EM table for a device + * @dev : Device for which the EM is to be updated + * @new_table : The new EM table that is going to be used from now + * + * Update EM runtime modifiable table for the @dev using the provided @table. + * + * This function uses a mutex to serialize writers, so it must not be called + * from a non-sleeping context. + * + * Return 0 on success or an error code on failure. + */ +int em_dev_update_perf_domain(struct device *dev, + struct em_perf_table __rcu *new_table) +{ + struct em_perf_table __rcu *old_table; + struct em_perf_domain *pd; + + if (!dev) + return -EINVAL; + + /* Serialize update/unregister or concurrent updates */ + mutex_lock(&em_pd_mutex); + + if (!dev->em_pd) { + mutex_unlock(&em_pd_mutex); + return -EINVAL; + } + pd = dev->em_pd; + + kref_get(&new_table->kref); + + old_table = pd->em_table; + rcu_assign_pointer(pd->em_table, new_table); + + em_cpufreq_update_efficiencies(dev, new_table->state); + + em_table_free(old_table); + + mutex_unlock(&em_pd_mutex); + return 0; +} +EXPORT_SYMBOL_GPL(em_dev_update_perf_domain); + +static int em_create_perf_table(struct device *dev, struct em_perf_domain *pd, + struct em_perf_state *table, + struct em_data_callback *cb, + unsigned long flags) +{ + unsigned long power, freq, prev_freq = 0; + int nr_states = pd->nr_perf_states; + int i, ret; + + /* Build the list of performance states for this performance domain */ + for (i = 0, freq = 0; i < nr_states; i++, freq++) { + /* + * active_power() is a driver callback which ceils 'freq' to + * lowest performance state of 'dev' above 'freq' and updates + * 'power' and 'freq' accordingly. + */ + ret = cb->active_power(dev, &power, &freq); + if (ret) { + dev_err(dev, "EM: invalid perf. state: %d\n", + ret); + return -EINVAL; + } + + /* + * We expect the driver callback to increase the frequency for + * higher performance states. + */ + if (freq <= prev_freq) { + dev_err(dev, "EM: non-increasing freq: %lu\n", + freq); + return -EINVAL; + } + + /* + * The power returned by active_state() is expected to be + * positive and be in range. + */ + if (!power || power > EM_MAX_POWER) { + dev_err(dev, "EM: invalid power: %lu\n", + power); + return -EINVAL; + } + + table[i].power = power; + table[i].frequency = prev_freq = freq; + } + + em_init_performance(dev, pd, table, nr_states); + + ret = em_compute_costs(dev, table, cb, nr_states, flags); + if (ret) + return -EINVAL; return 0; - -free_ps_table: - kfree(table); - return -EINVAL; } static int em_create_pd(struct device *dev, int nr_states, struct em_data_callback *cb, cpumask_t *cpus, unsigned long flags) { + struct em_perf_table __rcu *em_table; struct em_perf_domain *pd; struct device *cpu_dev; int cpu, ret, num_cpus; @@ -220,11 +424,17 @@ static int em_create_pd(struct device *dev, int nr_states, return -ENOMEM; } - ret = em_create_perf_table(dev, pd, nr_states, cb, flags); - if (ret) { - kfree(pd); - return ret; - } + pd->nr_perf_states = nr_states; + + em_table = em_table_alloc(pd); + if (!em_table) + goto free_pd; + + ret = em_create_perf_table(dev, pd, em_table->state, cb, flags); + if (ret) + goto free_pd_table; + + rcu_assign_pointer(pd->em_table, em_table); if (_is_cpu_device(dev)) for_each_cpu(cpu, cpus) { @@ -235,26 +445,37 @@ static int em_create_pd(struct device *dev, int nr_states, dev->em_pd = pd; return 0; + +free_pd_table: + kfree(em_table); +free_pd: + kfree(pd); + return -EINVAL; } -static void em_cpufreq_update_efficiencies(struct device *dev) +static void +em_cpufreq_update_efficiencies(struct device *dev, struct em_perf_state *table) { struct em_perf_domain *pd = dev->em_pd; - struct em_perf_state *table; struct cpufreq_policy *policy; int found = 0; - int i; + int i, cpu; - if (!_is_cpu_device(dev) || !pd) + if (!_is_cpu_device(dev)) return; - policy = cpufreq_cpu_get(cpumask_first(em_span_cpus(pd))); - if (!policy) { - dev_warn(dev, "EM: Access to CPUFreq policy failed"); + /* Try to get a CPU which is active and in this PD */ + cpu = cpumask_first_and(em_span_cpus(pd), cpu_active_mask); + if (cpu >= nr_cpu_ids) { + dev_warn(dev, "EM: No online CPU for CPUFreq policy\n"); return; } - table = pd->table; + policy = cpufreq_cpu_get(cpu); + if (!policy) { + dev_warn(dev, "EM: Access to CPUFreq policy failed\n"); + return; + } for (i = 0; i < pd->nr_perf_states; i++) { if (!(table[i].flags & EM_PERF_STATE_INEFFICIENT)) @@ -397,13 +618,17 @@ int em_dev_register_perf_domain(struct device *dev, unsigned int nr_states, dev->em_pd->flags |= flags; - em_cpufreq_update_efficiencies(dev); + em_cpufreq_update_efficiencies(dev, dev->em_pd->em_table->state); em_debug_create_pd(dev); dev_info(dev, "EM: created perf domain\n"); unlock: mutex_unlock(&em_pd_mutex); + + if (_is_cpu_device(dev)) + em_check_capacity_update(); + return ret; } EXPORT_SYMBOL_GPL(em_dev_register_perf_domain); @@ -430,9 +655,125 @@ void em_dev_unregister_perf_domain(struct device *dev) mutex_lock(&em_pd_mutex); em_debug_remove_pd(dev); - kfree(dev->em_pd->table); + em_table_free(dev->em_pd->em_table); + kfree(dev->em_pd); dev->em_pd = NULL; mutex_unlock(&em_pd_mutex); } EXPORT_SYMBOL_GPL(em_dev_unregister_perf_domain); + +/* + * Adjustment of CPU performance values after boot, when all CPUs capacites + * are correctly calculated. + */ +static void em_adjust_new_capacity(struct device *dev, + struct em_perf_domain *pd, + u64 max_cap) +{ + struct em_perf_table __rcu *em_table; + struct em_perf_state *ps, *new_ps; + int ret, ps_size; + + em_table = em_table_alloc(pd); + if (!em_table) { + dev_warn(dev, "EM: allocation failed\n"); + return; + } + + new_ps = em_table->state; + + rcu_read_lock(); + ps = em_perf_state_from_pd(pd); + /* Initialize data based on old table */ + ps_size = sizeof(struct em_perf_state) * pd->nr_perf_states; + memcpy(new_ps, ps, ps_size); + + rcu_read_unlock(); + + em_init_performance(dev, pd, new_ps, pd->nr_perf_states); + ret = em_compute_costs(dev, new_ps, NULL, pd->nr_perf_states, + pd->flags); + if (ret) { + dev_warn(dev, "EM: compute costs failed\n"); + return; + } + + ret = em_dev_update_perf_domain(dev, em_table); + if (ret) + dev_warn(dev, "EM: update failed %d\n", ret); + + /* + * This is one-time-update, so give up the ownership in this updater. + * The EM framework has incremented the usage counter and from now + * will keep the reference (then free the memory when needed). + */ + em_table_free(em_table); +} + +static void em_check_capacity_update(void) +{ + cpumask_var_t cpu_done_mask; + struct em_perf_state *table; + struct em_perf_domain *pd; + unsigned long cpu_capacity; + int cpu; + + if (!zalloc_cpumask_var(&cpu_done_mask, GFP_KERNEL)) { + pr_warn("no free memory\n"); + return; + } + + /* Check if CPUs capacity has changed than update EM */ + for_each_possible_cpu(cpu) { + struct cpufreq_policy *policy; + unsigned long em_max_perf; + struct device *dev; + + if (cpumask_test_cpu(cpu, cpu_done_mask)) + continue; + + policy = cpufreq_cpu_get(cpu); + if (!policy) { + pr_debug("Accessing cpu%d policy failed\n", cpu); + schedule_delayed_work(&em_update_work, + msecs_to_jiffies(1000)); + break; + } + cpufreq_cpu_put(policy); + + pd = em_cpu_get(cpu); + if (!pd || em_is_artificial(pd)) + continue; + + cpumask_or(cpu_done_mask, cpu_done_mask, + em_span_cpus(pd)); + + cpu_capacity = arch_scale_cpu_capacity(cpu); + + rcu_read_lock(); + table = em_perf_state_from_pd(pd); + em_max_perf = table[pd->nr_perf_states - 1].performance; + rcu_read_unlock(); + + /* + * Check if the CPU capacity has been adjusted during boot + * and trigger the update for new performance values. + */ + if (em_max_perf == cpu_capacity) + continue; + + pr_debug("updating cpu%d cpu_cap=%lu old capacity=%lu\n", + cpu, cpu_capacity, em_max_perf); + + dev = get_cpu_device(cpu); + em_adjust_new_capacity(dev, pd, cpu_capacity); + } + + free_cpumask_var(cpu_done_mask); +} + +static void em_update_workfn(struct work_struct *work) +{ + em_check_capacity_update(); +} diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index 4b0b7cf2e019..43b1a82e800c 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -47,6 +47,15 @@ dev_t swsusp_resume_device; sector_t swsusp_resume_block; __visible int in_suspend __nosavedata; +static char hibernate_compressor[CRYPTO_MAX_ALG_NAME] = CONFIG_HIBERNATION_DEF_COMP; + +/* + * Compression/decompression algorithm to be used while saving/loading + * image to/from disk. This would later be used in 'kernel/power/swap.c' + * to allocate comp streams. + */ +char hib_comp_algo[CRYPTO_MAX_ALG_NAME]; + enum { HIBERNATION_INVALID, HIBERNATION_PLATFORM, @@ -718,6 +727,9 @@ static int load_image_and_restore(void) return error; } +#define COMPRESSION_ALGO_LZO "lzo" +#define COMPRESSION_ALGO_LZ4 "lz4" + /** * hibernate - Carry out system hibernation, including saving the image. */ @@ -732,6 +744,17 @@ int hibernate(void) return -EPERM; } + /* + * Query for the compression algorithm support if compression is enabled. + */ + if (!nocompress) { + strscpy(hib_comp_algo, hibernate_compressor, sizeof(hib_comp_algo)); + if (crypto_has_comp(hib_comp_algo, 0, 0) != 1) { + pr_err("%s compression is not available\n", hib_comp_algo); + return -EOPNOTSUPP; + } + } + sleep_flags = lock_system_sleep(); /* The snapshot device should not be opened while we're running */ if (!hibernate_acquire()) { @@ -766,11 +789,24 @@ int hibernate(void) if (hibernation_mode == HIBERNATION_PLATFORM) flags |= SF_PLATFORM_MODE; - if (nocompress) + if (nocompress) { flags |= SF_NOCOMPRESS_MODE; - else + } else { flags |= SF_CRC32_MODE; + /* + * By default, LZO compression is enabled. Use SF_COMPRESSION_ALG_LZ4 + * to override this behaviour and use LZ4. + * + * Refer kernel/power/power.h for more details + */ + + if (!strcmp(hib_comp_algo, COMPRESSION_ALGO_LZ4)) + flags |= SF_COMPRESSION_ALG_LZ4; + else + flags |= SF_COMPRESSION_ALG_LZO; + } + pm_pr_dbg("Writing hibernation image.\n"); error = swsusp_write(flags); swsusp_free(); @@ -955,6 +991,22 @@ static int software_resume(void) if (error) goto Unlock; + /* + * Check if the hibernation image is compressed. If so, query for + * the algorithm support. + */ + if (!(swsusp_header_flags & SF_NOCOMPRESS_MODE)) { + if (swsusp_header_flags & SF_COMPRESSION_ALG_LZ4) + strscpy(hib_comp_algo, COMPRESSION_ALGO_LZ4, sizeof(hib_comp_algo)); + else + strscpy(hib_comp_algo, COMPRESSION_ALGO_LZO, sizeof(hib_comp_algo)); + if (crypto_has_comp(hib_comp_algo, 0, 0) != 1) { + pr_err("%s compression is not available\n", hib_comp_algo); + error = -EOPNOTSUPP; + goto Unlock; + } + } + /* The snapshot device should not be opened while we're running */ if (!hibernate_acquire()) { error = -EBUSY; @@ -1370,6 +1422,57 @@ static int __init nohibernate_setup(char *str) return 1; } +static const char * const comp_alg_enabled[] = { +#if IS_ENABLED(CONFIG_CRYPTO_LZO) + COMPRESSION_ALGO_LZO, +#endif +#if IS_ENABLED(CONFIG_CRYPTO_LZ4) + COMPRESSION_ALGO_LZ4, +#endif +}; + +static int hibernate_compressor_param_set(const char *compressor, + const struct kernel_param *kp) +{ + unsigned int sleep_flags; + int index, ret; + + sleep_flags = lock_system_sleep(); + + index = sysfs_match_string(comp_alg_enabled, compressor); + if (index >= 0) { + ret = param_set_copystring(comp_alg_enabled[index], kp); + if (!ret) + strscpy(hib_comp_algo, comp_alg_enabled[index], + sizeof(hib_comp_algo)); + } else { + ret = index; + } + + unlock_system_sleep(sleep_flags); + + if (ret) + pr_debug("Cannot set specified compressor %s\n", + compressor); + + return ret; +} + +static const struct kernel_param_ops hibernate_compressor_param_ops = { + .set = hibernate_compressor_param_set, + .get = param_get_string, +}; + +static struct kparam_string hibernate_compressor_param_string = { + .maxlen = sizeof(hibernate_compressor), + .string = hibernate_compressor, +}; + +module_param_cb(compressor, &hibernate_compressor_param_ops, + &hibernate_compressor_param_string, 0644); +MODULE_PARM_DESC(compressor, + "Compression algorithm to be used with hibernation"); + __setup("noresume", noresume_setup); __setup("resume_offset=", resume_offset_setup); __setup("resume=", resume_setup); diff --git a/kernel/power/main.c b/kernel/power/main.c index b1ae9b677d03..a9e0693aaf69 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -95,19 +95,6 @@ int unregister_pm_notifier(struct notifier_block *nb) } EXPORT_SYMBOL_GPL(unregister_pm_notifier); -void pm_report_hw_sleep_time(u64 t) -{ - suspend_stats.last_hw_sleep = t; - suspend_stats.total_hw_sleep += t; -} -EXPORT_SYMBOL_GPL(pm_report_hw_sleep_time); - -void pm_report_max_hw_sleep(u64 t) -{ - suspend_stats.max_hw_sleep = t; -} -EXPORT_SYMBOL_GPL(pm_report_max_hw_sleep); - int pm_notifier_call_chain_robust(unsigned long val_up, unsigned long val_down) { int ret; @@ -319,26 +306,86 @@ static ssize_t pm_test_store(struct kobject *kobj, struct kobj_attribute *attr, power_attr(pm_test); #endif /* CONFIG_PM_SLEEP_DEBUG */ -static char *suspend_step_name(enum suspend_stat_step step) +#define SUSPEND_NR_STEPS SUSPEND_RESUME +#define REC_FAILED_NUM 2 + +struct suspend_stats { + unsigned int step_failures[SUSPEND_NR_STEPS]; + unsigned int success; + unsigned int fail; + int last_failed_dev; + char failed_devs[REC_FAILED_NUM][40]; + int last_failed_errno; + int errno[REC_FAILED_NUM]; + int last_failed_step; + u64 last_hw_sleep; + u64 total_hw_sleep; + u64 max_hw_sleep; + enum suspend_stat_step failed_steps[REC_FAILED_NUM]; +}; + +static struct suspend_stats suspend_stats; +static DEFINE_MUTEX(suspend_stats_lock); + +void dpm_save_failed_dev(const char *name) { - switch (step) { - case SUSPEND_FREEZE: - return "freeze"; - case SUSPEND_PREPARE: - return "prepare"; - case SUSPEND_SUSPEND: - return "suspend"; - case SUSPEND_SUSPEND_NOIRQ: - return "suspend_noirq"; - case SUSPEND_RESUME_NOIRQ: - return "resume_noirq"; - case SUSPEND_RESUME: - return "resume"; - default: - return ""; - } + mutex_lock(&suspend_stats_lock); + + strscpy(suspend_stats.failed_devs[suspend_stats.last_failed_dev], + name, sizeof(suspend_stats.failed_devs[0])); + suspend_stats.last_failed_dev++; + suspend_stats.last_failed_dev %= REC_FAILED_NUM; + + mutex_unlock(&suspend_stats_lock); } +void dpm_save_failed_step(enum suspend_stat_step step) +{ + suspend_stats.step_failures[step-1]++; + suspend_stats.failed_steps[suspend_stats.last_failed_step] = step; + suspend_stats.last_failed_step++; + suspend_stats.last_failed_step %= REC_FAILED_NUM; +} + +void dpm_save_errno(int err) +{ + if (!err) { + suspend_stats.success++; + return; + } + + suspend_stats.fail++; + + suspend_stats.errno[suspend_stats.last_failed_errno] = err; + suspend_stats.last_failed_errno++; + suspend_stats.last_failed_errno %= REC_FAILED_NUM; +} + +void pm_report_hw_sleep_time(u64 t) +{ + suspend_stats.last_hw_sleep = t; + suspend_stats.total_hw_sleep += t; +} +EXPORT_SYMBOL_GPL(pm_report_hw_sleep_time); + +void pm_report_max_hw_sleep(u64 t) +{ + suspend_stats.max_hw_sleep = t; +} +EXPORT_SYMBOL_GPL(pm_report_max_hw_sleep); + +static const char * const suspend_step_names[] = { + [SUSPEND_WORKING] = "", + [SUSPEND_FREEZE] = "freeze", + [SUSPEND_PREPARE] = "prepare", + [SUSPEND_SUSPEND] = "suspend", + [SUSPEND_SUSPEND_LATE] = "suspend_late", + [SUSPEND_SUSPEND_NOIRQ] = "suspend_noirq", + [SUSPEND_RESUME_NOIRQ] = "resume_noirq", + [SUSPEND_RESUME_EARLY] = "resume_early", + [SUSPEND_RESUME] = "resume", +}; + #define suspend_attr(_name, format_str) \ static ssize_t _name##_show(struct kobject *kobj, \ struct kobj_attribute *attr, char *buf) \ @@ -347,20 +394,30 @@ static ssize_t _name##_show(struct kobject *kobj, \ } \ static struct kobj_attribute _name = __ATTR_RO(_name) -suspend_attr(success, "%d\n"); -suspend_attr(fail, "%d\n"); -suspend_attr(failed_freeze, "%d\n"); -suspend_attr(failed_prepare, "%d\n"); -suspend_attr(failed_suspend, "%d\n"); -suspend_attr(failed_suspend_late, "%d\n"); -suspend_attr(failed_suspend_noirq, "%d\n"); -suspend_attr(failed_resume, "%d\n"); -suspend_attr(failed_resume_early, "%d\n"); -suspend_attr(failed_resume_noirq, "%d\n"); +suspend_attr(success, "%u\n"); +suspend_attr(fail, "%u\n"); suspend_attr(last_hw_sleep, "%llu\n"); suspend_attr(total_hw_sleep, "%llu\n"); suspend_attr(max_hw_sleep, "%llu\n"); +#define suspend_step_attr(_name, step) \ +static ssize_t _name##_show(struct kobject *kobj, \ + struct kobj_attribute *attr, char *buf) \ +{ \ + return sprintf(buf, "%u\n", \ + suspend_stats.step_failures[step-1]); \ +} \ +static struct kobj_attribute _name = __ATTR_RO(_name) + +suspend_step_attr(failed_freeze, SUSPEND_FREEZE); +suspend_step_attr(failed_prepare, SUSPEND_PREPARE); +suspend_step_attr(failed_suspend, SUSPEND_SUSPEND); +suspend_step_attr(failed_suspend_late, SUSPEND_SUSPEND_LATE); +suspend_step_attr(failed_suspend_noirq, SUSPEND_SUSPEND_NOIRQ); +suspend_step_attr(failed_resume, SUSPEND_RESUME); +suspend_step_attr(failed_resume_early, SUSPEND_RESUME_EARLY); +suspend_step_attr(failed_resume_noirq, SUSPEND_RESUME_NOIRQ); + static ssize_t last_failed_dev_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -392,16 +449,14 @@ static struct kobj_attribute last_failed_errno = __ATTR_RO(last_failed_errno); static ssize_t last_failed_step_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { - int index; enum suspend_stat_step step; - char *last_failed_step = NULL; + int index; index = suspend_stats.last_failed_step + REC_FAILED_NUM - 1; index %= REC_FAILED_NUM; step = suspend_stats.failed_steps[index]; - last_failed_step = suspend_step_name(step); - return sprintf(buf, "%s\n", last_failed_step); + return sprintf(buf, "%s\n", suspend_step_names[step]); } static struct kobj_attribute last_failed_step = __ATTR_RO(last_failed_step); @@ -449,6 +504,7 @@ static const struct attribute_group suspend_attr_group = { static int suspend_stats_show(struct seq_file *s, void *unused) { int i, index, last_dev, last_errno, last_step; + enum suspend_stat_step step; last_dev = suspend_stats.last_failed_dev + REC_FAILED_NUM - 1; last_dev %= REC_FAILED_NUM; @@ -456,47 +512,35 @@ static int suspend_stats_show(struct seq_file *s, void *unused) last_errno %= REC_FAILED_NUM; last_step = suspend_stats.last_failed_step + REC_FAILED_NUM - 1; last_step %= REC_FAILED_NUM; - seq_printf(s, "%s: %d\n%s: %d\n%s: %d\n%s: %d\n%s: %d\n" - "%s: %d\n%s: %d\n%s: %d\n%s: %d\n%s: %d\n", - "success", suspend_stats.success, - "fail", suspend_stats.fail, - "failed_freeze", suspend_stats.failed_freeze, - "failed_prepare", suspend_stats.failed_prepare, - "failed_suspend", suspend_stats.failed_suspend, - "failed_suspend_late", - suspend_stats.failed_suspend_late, - "failed_suspend_noirq", - suspend_stats.failed_suspend_noirq, - "failed_resume", suspend_stats.failed_resume, - "failed_resume_early", - suspend_stats.failed_resume_early, - "failed_resume_noirq", - suspend_stats.failed_resume_noirq); + + seq_printf(s, "success: %u\nfail: %u\n", + suspend_stats.success, suspend_stats.fail); + + for (step = SUSPEND_FREEZE; step <= SUSPEND_NR_STEPS; step++) + seq_printf(s, "failed_%s: %u\n", suspend_step_names[step], + suspend_stats.step_failures[step-1]); + seq_printf(s, "failures:\n last_failed_dev:\t%-s\n", - suspend_stats.failed_devs[last_dev]); + suspend_stats.failed_devs[last_dev]); for (i = 1; i < REC_FAILED_NUM; i++) { index = last_dev + REC_FAILED_NUM - i; index %= REC_FAILED_NUM; - seq_printf(s, "\t\t\t%-s\n", - suspend_stats.failed_devs[index]); + seq_printf(s, "\t\t\t%-s\n", suspend_stats.failed_devs[index]); } seq_printf(s, " last_failed_errno:\t%-d\n", suspend_stats.errno[last_errno]); for (i = 1; i < REC_FAILED_NUM; i++) { index = last_errno + REC_FAILED_NUM - i; index %= REC_FAILED_NUM; - seq_printf(s, "\t\t\t%-d\n", - suspend_stats.errno[index]); + seq_printf(s, "\t\t\t%-d\n", suspend_stats.errno[index]); } seq_printf(s, " last_failed_step:\t%-s\n", - suspend_step_name( - suspend_stats.failed_steps[last_step])); + suspend_step_names[suspend_stats.failed_steps[last_step]]); for (i = 1; i < REC_FAILED_NUM; i++) { index = last_step + REC_FAILED_NUM - i; index %= REC_FAILED_NUM; seq_printf(s, "\t\t\t%-s\n", - suspend_step_name( - suspend_stats.failed_steps[index])); + suspend_step_names[suspend_stats.failed_steps[index]]); } return 0; diff --git a/kernel/power/power.h b/kernel/power/power.h index 8499a39c62f4..de0e6b1077f2 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -6,6 +6,7 @@ #include #include #include +#include struct swsusp_info { struct new_utsname uts; @@ -54,6 +55,10 @@ asmlinkage int swsusp_save(void); /* kernel/power/hibernate.c */ extern bool freezer_test_done; +extern char hib_comp_algo[CRYPTO_MAX_ALG_NAME]; + +/* kernel/power/swap.c */ +extern unsigned int swsusp_header_flags; extern int hibernation_snapshot(int platform_mode); extern int hibernation_restore(int platform_mode); @@ -148,7 +153,7 @@ extern unsigned int snapshot_additional_pages(struct zone *zone); extern unsigned long snapshot_get_image_size(void); extern int snapshot_read_next(struct snapshot_handle *handle); extern int snapshot_write_next(struct snapshot_handle *handle); -extern void snapshot_write_finalize(struct snapshot_handle *handle); +int snapshot_write_finalize(struct snapshot_handle *handle); extern int snapshot_image_loaded(struct snapshot_handle *handle); extern bool hibernate_acquire(void); @@ -162,11 +167,25 @@ extern int swsusp_swap_in_use(void); * Flags that can be passed from the hibernatig hernel to the "boot" kernel in * the image header. */ +#define SF_COMPRESSION_ALG_LZO 0 /* dummy, details given below */ #define SF_PLATFORM_MODE 1 #define SF_NOCOMPRESS_MODE 2 #define SF_CRC32_MODE 4 #define SF_HW_SIG 8 +/* + * Bit to indicate the compression algorithm to be used(for LZ4). The same + * could be checked while saving/loading image to/from disk to use the + * corresponding algorithms. + * + * By default, LZO compression is enabled if SF_CRC32_MODE is set. Use + * SF_COMPRESSION_ALG_LZ4 to override this behaviour and use LZ4. + * + * SF_CRC32_MODE, SF_COMPRESSION_ALG_LZO(dummy) -> Compression, LZO + * SF_CRC32_MODE, SF_COMPRESSION_ALG_LZ4 -> Compression, LZ4 + */ +#define SF_COMPRESSION_ALG_LZ4 16 + /* kernel/power/hibernate.c */ int swsusp_check(bool exclusive); extern void swsusp_free(void); @@ -327,3 +346,5 @@ static inline void pm_sleep_enable_secondary_cpus(void) suspend_enable_secondary_cpus(); cpuidle_resume(); } + +void dpm_save_errno(int err); diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index 5c96ff067c64..405eddbda4fc 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -58,22 +58,24 @@ static inline void hibernate_restore_protection_end(void) hibernate_restore_protection_active = false; } -static inline void hibernate_restore_protect_page(void *page_address) +static inline int __must_check hibernate_restore_protect_page(void *page_address) { if (hibernate_restore_protection_active) - set_memory_ro((unsigned long)page_address, 1); + return set_memory_ro((unsigned long)page_address, 1); + return 0; } -static inline void hibernate_restore_unprotect_page(void *page_address) +static inline int hibernate_restore_unprotect_page(void *page_address) { if (hibernate_restore_protection_active) - set_memory_rw((unsigned long)page_address, 1); + return set_memory_rw((unsigned long)page_address, 1); + return 0; } #else static inline void hibernate_restore_protection_begin(void) {} static inline void hibernate_restore_protection_end(void) {} -static inline void hibernate_restore_protect_page(void *page_address) {} -static inline void hibernate_restore_unprotect_page(void *page_address) {} +static inline int __must_check hibernate_restore_protect_page(void *page_address) {return 0; } +static inline int hibernate_restore_unprotect_page(void *page_address) {return 0; } #endif /* CONFIG_STRICT_KERNEL_RWX && CONFIG_ARCH_HAS_SET_MEMORY */ @@ -2832,7 +2834,9 @@ next: } } else { copy_last_highmem_page(); - hibernate_restore_protect_page(handle->buffer); + error = hibernate_restore_protect_page(handle->buffer); + if (error) + return error; handle->buffer = get_buffer(&orig_bm, &ca); if (IS_ERR(handle->buffer)) return PTR_ERR(handle->buffer); @@ -2858,15 +2862,18 @@ next: * stored in highmem. Additionally, it recycles bitmap memory that's not * necessary any more. */ -void snapshot_write_finalize(struct snapshot_handle *handle) +int snapshot_write_finalize(struct snapshot_handle *handle) { + int error; + copy_last_highmem_page(); - hibernate_restore_protect_page(handle->buffer); + error = hibernate_restore_protect_page(handle->buffer); /* Do that only if we have loaded the image entirely */ if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages + nr_zero_pages) { memory_bm_recycle(&orig_bm); free_highmem_data(); } + return error; } int snapshot_image_loaded(struct snapshot_handle *handle) diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c index c61c378e514f..9d4375ed43ee 100644 --- a/kernel/power/suspend.c +++ b/kernel/power/suspend.c @@ -195,6 +195,7 @@ static int __init mem_sleep_default_setup(char *str) if (mem_sleep_labels[state] && !strcmp(str, mem_sleep_labels[state])) { mem_sleep_default = state; + mem_sleep_current = state; break; } @@ -371,7 +372,6 @@ static int suspend_prepare(suspend_state_t state) return 0; log_suspend_abort_reason("One or more tasks refusing to freeze"); - suspend_stats.failed_freeze++; dpm_save_failed_step(SUSPEND_FREEZE); pm_notifier_call_chain(PM_POST_SUSPEND); Restore: @@ -633,12 +633,7 @@ int pm_suspend(suspend_state_t state) pr_info("suspend entry (%s)\n", mem_sleep_labels[state]); error = enter_state(state); - if (error) { - suspend_stats.fail++; - dpm_save_failed_errno(error); - } else { - suspend_stats.success++; - } + dpm_save_errno(error); pr_info("suspend exit\n"); return error; } diff --git a/kernel/power/swap.c b/kernel/power/swap.c index 692f12fe60c1..5bc04bfe2db1 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -23,7 +23,6 @@ #include #include #include -#include #include #include #include @@ -339,6 +338,13 @@ static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags) return error; } +/* + * Hold the swsusp_header flag. This is used in software_resume() in + * 'kernel/power/hibernate' to check if the image is compressed and query + * for the compression algorithm support(if so). + */ +unsigned int swsusp_header_flags; + /** * swsusp_swap_check - check if the resume device is a swap device * and get its index (if so) @@ -514,25 +520,30 @@ static int swap_writer_finish(struct swap_map_handle *handle, return error; } +/* + * Bytes we need for compressed data in worst case. We assume(limitation) + * this is the worst of all the compression algorithms. + */ +#define bytes_worst_compress(x) ((x) + ((x) / 16) + 64 + 3 + 2) + /* We need to remember how much compressed data we need to read. */ -#define LZO_HEADER sizeof(size_t) +#define CMP_HEADER sizeof(size_t) /* Number of pages/bytes we'll compress at one time. */ -#define LZO_UNC_PAGES 32 -#define LZO_UNC_SIZE (LZO_UNC_PAGES * PAGE_SIZE) +#define UNC_PAGES 32 +#define UNC_SIZE (UNC_PAGES * PAGE_SIZE) -/* Number of pages/bytes we need for compressed data (worst case). */ -#define LZO_CMP_PAGES DIV_ROUND_UP(lzo1x_worst_compress(LZO_UNC_SIZE) + \ - LZO_HEADER, PAGE_SIZE) -#define LZO_CMP_SIZE (LZO_CMP_PAGES * PAGE_SIZE) +/* Number of pages we need for compressed data (worst case). */ +#define CMP_PAGES DIV_ROUND_UP(bytes_worst_compress(UNC_SIZE) + \ + CMP_HEADER, PAGE_SIZE) +#define CMP_SIZE (CMP_PAGES * PAGE_SIZE) /* Maximum number of threads for compression/decompression. */ -#define LZO_THREADS 3 +#define CMP_THREADS 3 /* Minimum/maximum number of pages for read buffering. */ -#define LZO_MIN_RD_PAGES 1024 -#define LZO_MAX_RD_PAGES 8192 - +#define CMP_MIN_RD_PAGES 1024 +#define CMP_MAX_RD_PAGES 8192 /** * save_image - save the suspend image data @@ -593,8 +604,8 @@ struct crc_data { wait_queue_head_t go; /* start crc update */ wait_queue_head_t done; /* crc update done */ u32 *crc32; /* points to handle's crc32 */ - size_t *unc_len[LZO_THREADS]; /* uncompressed lengths */ - unsigned char *unc[LZO_THREADS]; /* uncompressed data */ + size_t *unc_len[CMP_THREADS]; /* uncompressed lengths */ + unsigned char *unc[CMP_THREADS]; /* uncompressed data */ }; /* @@ -625,10 +636,11 @@ static int crc32_threadfn(void *data) return 0; } /* - * Structure used for LZO data compression. + * Structure used for data compression. */ struct cmp_data { struct task_struct *thr; /* thread */ + struct crypto_comp *cc; /* crypto compressor stream */ atomic_t ready; /* ready to start flag */ atomic_t stop; /* ready to stop flag */ int ret; /* return code */ @@ -636,17 +648,20 @@ struct cmp_data { wait_queue_head_t done; /* compression done */ size_t unc_len; /* uncompressed length */ size_t cmp_len; /* compressed length */ - unsigned char unc[LZO_UNC_SIZE]; /* uncompressed buffer */ - unsigned char cmp[LZO_CMP_SIZE]; /* compressed buffer */ - unsigned char wrk[LZO1X_1_MEM_COMPRESS]; /* compression workspace */ + unsigned char unc[UNC_SIZE]; /* uncompressed buffer */ + unsigned char cmp[CMP_SIZE]; /* compressed buffer */ }; +/* Indicates the image size after compression */ +static atomic_t compressed_size = ATOMIC_INIT(0); + /* * Compression function that runs in its own thread. */ -static int lzo_compress_threadfn(void *data) +static int compress_threadfn(void *data) { struct cmp_data *d = data; + unsigned int cmp_len = 0; while (1) { wait_event(d->go, atomic_read_acquire(&d->ready) || @@ -660,9 +675,13 @@ static int lzo_compress_threadfn(void *data) } atomic_set(&d->ready, 0); - d->ret = lzo1x_1_compress(d->unc, d->unc_len, - d->cmp + LZO_HEADER, &d->cmp_len, - d->wrk); + cmp_len = CMP_SIZE - CMP_HEADER; + d->ret = crypto_comp_compress(d->cc, d->unc, d->unc_len, + d->cmp + CMP_HEADER, + &cmp_len); + d->cmp_len = cmp_len; + + atomic_set(&compressed_size, atomic_read(&compressed_size) + d->cmp_len); atomic_set_release(&d->stop, 1); wake_up(&d->done); } @@ -670,14 +689,14 @@ static int lzo_compress_threadfn(void *data) } /** - * save_image_lzo - Save the suspend image data compressed with LZO. + * save_compressed_image - Save the suspend image data after compression. * @handle: Swap map handle to use for saving the image. * @snapshot: Image to read data from. * @nr_to_write: Number of pages to save. */ -static int save_image_lzo(struct swap_map_handle *handle, - struct snapshot_handle *snapshot, - unsigned int nr_to_write) +static int save_compressed_image(struct swap_map_handle *handle, + struct snapshot_handle *snapshot, + unsigned int nr_to_write) { unsigned int m; int ret = 0; @@ -694,23 +713,25 @@ static int save_image_lzo(struct swap_map_handle *handle, hib_init_batch(&hb); + atomic_set(&compressed_size, 0); + /* * We'll limit the number of threads for compression to limit memory * footprint. */ nr_threads = num_online_cpus() - 1; - nr_threads = clamp_val(nr_threads, 1, LZO_THREADS); + nr_threads = clamp_val(nr_threads, 1, CMP_THREADS); page = (void *)__get_free_page(GFP_NOIO | __GFP_HIGH); if (!page) { - pr_err("Failed to allocate LZO page\n"); + pr_err("Failed to allocate %s page\n", hib_comp_algo); ret = -ENOMEM; goto out_clean; } data = vzalloc(array_size(nr_threads, sizeof(*data))); if (!data) { - pr_err("Failed to allocate LZO data\n"); + pr_err("Failed to allocate %s data\n", hib_comp_algo); ret = -ENOMEM; goto out_clean; } @@ -729,7 +750,14 @@ static int save_image_lzo(struct swap_map_handle *handle, init_waitqueue_head(&data[thr].go); init_waitqueue_head(&data[thr].done); - data[thr].thr = kthread_run(lzo_compress_threadfn, + data[thr].cc = crypto_alloc_comp(hib_comp_algo, 0, 0); + if (IS_ERR_OR_NULL(data[thr].cc)) { + pr_err("Could not allocate comp stream %ld\n", PTR_ERR(data[thr].cc)); + ret = -EFAULT; + goto out_clean; + } + + data[thr].thr = kthread_run(compress_threadfn, &data[thr], "image_compress/%u", thr); if (IS_ERR(data[thr].thr)) { @@ -767,7 +795,7 @@ static int save_image_lzo(struct swap_map_handle *handle, */ handle->reqd_free_pages = reqd_free_pages(); - pr_info("Using %u thread(s) for compression\n", nr_threads); + pr_info("Using %u thread(s) for %s compression\n", nr_threads, hib_comp_algo); pr_info("Compressing and saving image data (%u pages)...\n", nr_to_write); m = nr_to_write / 10; @@ -777,7 +805,7 @@ static int save_image_lzo(struct swap_map_handle *handle, start = ktime_get(); for (;;) { for (thr = 0; thr < nr_threads; thr++) { - for (off = 0; off < LZO_UNC_SIZE; off += PAGE_SIZE) { + for (off = 0; off < UNC_SIZE; off += PAGE_SIZE) { ret = snapshot_read_next(snapshot); if (ret < 0) goto out_finish; @@ -817,14 +845,14 @@ static int save_image_lzo(struct swap_map_handle *handle, ret = data[thr].ret; if (ret < 0) { - pr_err("LZO compression failed\n"); + pr_err("%s compression failed\n", hib_comp_algo); goto out_finish; } if (unlikely(!data[thr].cmp_len || data[thr].cmp_len > - lzo1x_worst_compress(data[thr].unc_len))) { - pr_err("Invalid LZO compressed length\n"); + bytes_worst_compress(data[thr].unc_len))) { + pr_err("Invalid %s compressed length\n", hib_comp_algo); ret = -1; goto out_finish; } @@ -840,7 +868,7 @@ static int save_image_lzo(struct swap_map_handle *handle, * read it. */ for (off = 0; - off < LZO_HEADER + data[thr].cmp_len; + off < CMP_HEADER + data[thr].cmp_len; off += PAGE_SIZE) { memcpy(page, data[thr].cmp + off, PAGE_SIZE); @@ -862,6 +890,9 @@ out_finish: if (!ret) pr_info("Image saving done\n"); swsusp_show_speed(start, stop, nr_to_write, "Wrote"); + pr_info("Image size after compression: %d kbytes\n", + (atomic_read(&compressed_size) / 1024)); + out_clean: hib_finish_batch(&hb); if (crc) { @@ -870,9 +901,12 @@ out_clean: kfree(crc); } if (data) { - for (thr = 0; thr < nr_threads; thr++) + for (thr = 0; thr < nr_threads; thr++) { if (data[thr].thr) kthread_stop(data[thr].thr); + if (data[thr].cc) + crypto_free_comp(data[thr].cc); + } vfree(data); } if (page) free_page((unsigned long)page); @@ -942,7 +976,7 @@ int swsusp_write(unsigned int flags) if (!error) { error = (flags & SF_NOCOMPRESS_MODE) ? save_image(&handle, &snapshot, pages - 1) : - save_image_lzo(&handle, &snapshot, pages - 1); + save_compressed_image(&handle, &snapshot, pages - 1); } out_finish: error = swap_writer_finish(&handle, flags, error); @@ -1100,8 +1134,8 @@ static int load_image(struct swap_map_handle *handle, ret = err2; if (!ret) { pr_info("Image loading done\n"); - snapshot_write_finalize(snapshot); - if (!snapshot_image_loaded(snapshot)) + ret = snapshot_write_finalize(snapshot); + if (!ret && !snapshot_image_loaded(snapshot)) ret = -ENODATA; } swsusp_show_speed(start, stop, nr_to_read, "Read"); @@ -1109,10 +1143,11 @@ static int load_image(struct swap_map_handle *handle, } /* - * Structure used for LZO data decompression. + * Structure used for data decompression. */ struct dec_data { struct task_struct *thr; /* thread */ + struct crypto_comp *cc; /* crypto compressor stream */ atomic_t ready; /* ready to start flag */ atomic_t stop; /* ready to stop flag */ int ret; /* return code */ @@ -1120,16 +1155,17 @@ struct dec_data { wait_queue_head_t done; /* decompression done */ size_t unc_len; /* uncompressed length */ size_t cmp_len; /* compressed length */ - unsigned char unc[LZO_UNC_SIZE]; /* uncompressed buffer */ - unsigned char cmp[LZO_CMP_SIZE]; /* compressed buffer */ + unsigned char unc[UNC_SIZE]; /* uncompressed buffer */ + unsigned char cmp[CMP_SIZE]; /* compressed buffer */ }; /* * Decompression function that runs in its own thread. */ -static int lzo_decompress_threadfn(void *data) +static int decompress_threadfn(void *data) { struct dec_data *d = data; + unsigned int unc_len = 0; while (1) { wait_event(d->go, atomic_read_acquire(&d->ready) || @@ -1143,9 +1179,11 @@ static int lzo_decompress_threadfn(void *data) } atomic_set(&d->ready, 0); - d->unc_len = LZO_UNC_SIZE; - d->ret = lzo1x_decompress_safe(d->cmp + LZO_HEADER, d->cmp_len, - d->unc, &d->unc_len); + unc_len = UNC_SIZE; + d->ret = crypto_comp_decompress(d->cc, d->cmp + CMP_HEADER, d->cmp_len, + d->unc, &unc_len); + d->unc_len = unc_len; + if (clean_pages_on_decompress) flush_icache_range((unsigned long)d->unc, (unsigned long)d->unc + d->unc_len); @@ -1157,14 +1195,14 @@ static int lzo_decompress_threadfn(void *data) } /** - * load_image_lzo - Load compressed image data and decompress them with LZO. + * load_compressed_image - Load compressed image data and decompress it. * @handle: Swap map handle to use for loading data. * @snapshot: Image to copy uncompressed data into. * @nr_to_read: Number of pages to load. */ -static int load_image_lzo(struct swap_map_handle *handle, - struct snapshot_handle *snapshot, - unsigned int nr_to_read) +static int load_compressed_image(struct swap_map_handle *handle, + struct snapshot_handle *snapshot, + unsigned int nr_to_read) { unsigned int m; int ret = 0; @@ -1189,18 +1227,18 @@ static int load_image_lzo(struct swap_map_handle *handle, * footprint. */ nr_threads = num_online_cpus() - 1; - nr_threads = clamp_val(nr_threads, 1, LZO_THREADS); + nr_threads = clamp_val(nr_threads, 1, CMP_THREADS); - page = vmalloc(array_size(LZO_MAX_RD_PAGES, sizeof(*page))); + page = vmalloc(array_size(CMP_MAX_RD_PAGES, sizeof(*page))); if (!page) { - pr_err("Failed to allocate LZO page\n"); + pr_err("Failed to allocate %s page\n", hib_comp_algo); ret = -ENOMEM; goto out_clean; } data = vzalloc(array_size(nr_threads, sizeof(*data))); if (!data) { - pr_err("Failed to allocate LZO data\n"); + pr_err("Failed to allocate %s data\n", hib_comp_algo); ret = -ENOMEM; goto out_clean; } @@ -1221,7 +1259,14 @@ static int load_image_lzo(struct swap_map_handle *handle, init_waitqueue_head(&data[thr].go); init_waitqueue_head(&data[thr].done); - data[thr].thr = kthread_run(lzo_decompress_threadfn, + data[thr].cc = crypto_alloc_comp(hib_comp_algo, 0, 0); + if (IS_ERR_OR_NULL(data[thr].cc)) { + pr_err("Could not allocate comp stream %ld\n", PTR_ERR(data[thr].cc)); + ret = -EFAULT; + goto out_clean; + } + + data[thr].thr = kthread_run(decompress_threadfn, &data[thr], "image_decompress/%u", thr); if (IS_ERR(data[thr].thr)) { @@ -1262,18 +1307,18 @@ static int load_image_lzo(struct swap_map_handle *handle, */ if (low_free_pages() > snapshot_get_image_size()) read_pages = (low_free_pages() - snapshot_get_image_size()) / 2; - read_pages = clamp_val(read_pages, LZO_MIN_RD_PAGES, LZO_MAX_RD_PAGES); + read_pages = clamp_val(read_pages, CMP_MIN_RD_PAGES, CMP_MAX_RD_PAGES); for (i = 0; i < read_pages; i++) { - page[i] = (void *)__get_free_page(i < LZO_CMP_PAGES ? + page[i] = (void *)__get_free_page(i < CMP_PAGES ? GFP_NOIO | __GFP_HIGH : GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY); if (!page[i]) { - if (i < LZO_CMP_PAGES) { + if (i < CMP_PAGES) { ring_size = i; - pr_err("Failed to allocate LZO pages\n"); + pr_err("Failed to allocate %s pages\n", hib_comp_algo); ret = -ENOMEM; goto out_clean; } else { @@ -1283,7 +1328,7 @@ static int load_image_lzo(struct swap_map_handle *handle, } want = ring_size = i; - pr_info("Using %u thread(s) for decompression\n", nr_threads); + pr_info("Using %u thread(s) for %s decompression\n", nr_threads, hib_comp_algo); pr_info("Loading and decompressing image data (%u pages)...\n", nr_to_read); m = nr_to_read / 10; @@ -1344,13 +1389,13 @@ static int load_image_lzo(struct swap_map_handle *handle, data[thr].cmp_len = *(size_t *)page[pg]; if (unlikely(!data[thr].cmp_len || data[thr].cmp_len > - lzo1x_worst_compress(LZO_UNC_SIZE))) { - pr_err("Invalid LZO compressed length\n"); + bytes_worst_compress(UNC_SIZE))) { + pr_err("Invalid %s compressed length\n", hib_comp_algo); ret = -1; goto out_finish; } - need = DIV_ROUND_UP(data[thr].cmp_len + LZO_HEADER, + need = DIV_ROUND_UP(data[thr].cmp_len + CMP_HEADER, PAGE_SIZE); if (need > have) { if (eof > 1) { @@ -1361,7 +1406,7 @@ static int load_image_lzo(struct swap_map_handle *handle, } for (off = 0; - off < LZO_HEADER + data[thr].cmp_len; + off < CMP_HEADER + data[thr].cmp_len; off += PAGE_SIZE) { memcpy(data[thr].cmp + off, page[pg], PAGE_SIZE); @@ -1378,7 +1423,7 @@ static int load_image_lzo(struct swap_map_handle *handle, /* * Wait for more data while we are decompressing. */ - if (have < LZO_CMP_PAGES && asked) { + if (have < CMP_PAGES && asked) { ret = hib_wait_io(&hb); if (ret) goto out_finish; @@ -1396,14 +1441,14 @@ static int load_image_lzo(struct swap_map_handle *handle, ret = data[thr].ret; if (ret < 0) { - pr_err("LZO decompression failed\n"); + pr_err("%s decompression failed\n", hib_comp_algo); goto out_finish; } if (unlikely(!data[thr].unc_len || - data[thr].unc_len > LZO_UNC_SIZE || - data[thr].unc_len & (PAGE_SIZE - 1))) { - pr_err("Invalid LZO uncompressed length\n"); + data[thr].unc_len > UNC_SIZE || + data[thr].unc_len & (PAGE_SIZE - 1))) { + pr_err("Invalid %s uncompressed length\n", hib_comp_algo); ret = -1; goto out_finish; } @@ -1441,8 +1486,8 @@ out_finish: stop = ktime_get(); if (!ret) { pr_info("Image loading done\n"); - snapshot_write_finalize(snapshot); - if (!snapshot_image_loaded(snapshot)) + ret = snapshot_write_finalize(snapshot); + if (!ret && !snapshot_image_loaded(snapshot)) ret = -ENODATA; if (!ret) { if (swsusp_header->flags & SF_CRC32_MODE) { @@ -1464,9 +1509,12 @@ out_clean: kfree(crc); } if (data) { - for (thr = 0; thr < nr_threads; thr++) + for (thr = 0; thr < nr_threads; thr++) { if (data[thr].thr) kthread_stop(data[thr].thr); + if (data[thr].cc) + crypto_free_comp(data[thr].cc); + } vfree(data); } vfree(page); @@ -1500,7 +1548,7 @@ int swsusp_read(unsigned int *flags_p) if (!error) { error = (*flags_p & SF_NOCOMPRESS_MODE) ? load_image(&handle, &snapshot, header->pages - 1) : - load_image_lzo(&handle, &snapshot, header->pages - 1); + load_compressed_image(&handle, &snapshot, header->pages - 1); } swap_reader_finish(&handle); end: @@ -1535,6 +1583,7 @@ int swsusp_check(bool exclusive) if (!memcmp(HIBERNATE_SIG, swsusp_header->sig, 10)) { memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10); + swsusp_header_flags = swsusp_header->flags; /* Reset swap signature now */ error = hib_submit_io(REQ_OP_WRITE | REQ_SYNC, swsusp_resume_block, diff --git a/kernel/power/user.c b/kernel/power/user.c index 3a4e70366f35..3aa41ba22129 100644 --- a/kernel/power/user.c +++ b/kernel/power/user.c @@ -317,7 +317,9 @@ static long snapshot_ioctl(struct file *filp, unsigned int cmd, break; case SNAPSHOT_ATOMIC_RESTORE: - snapshot_write_finalize(&data->handle); + error = snapshot_write_finalize(&data->handle); + if (error) + break; if (data->mode != O_WRONLY || !data->frozen || !snapshot_image_loaded(&data->handle)) { error = -EPERM; diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index b96077152f49..c8093bcc01fe 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -140,39 +140,6 @@ static inline bool nbcon_state_try_cmpxchg(struct console *con, struct nbcon_sta return atomic_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_state), &cur->atom, new->atom); } -#ifdef CONFIG_64BIT - -#define __seq_to_nbcon_seq(seq) (seq) -#define __nbcon_seq_to_seq(seq) (seq) - -#else /* CONFIG_64BIT */ - -#define __seq_to_nbcon_seq(seq) ((u32)seq) - -static inline u64 __nbcon_seq_to_seq(u32 nbcon_seq) -{ - u64 seq; - u64 rb_next_seq; - - /* - * The provided sequence is only the lower 32 bits of the ringbuffer - * sequence. It needs to be expanded to 64bit. Get the next sequence - * number from the ringbuffer and fold it. - * - * Having a 32bit representation in the console is sufficient. - * If a console ever gets more than 2^31 records behind - * the ringbuffer then this is the least of the problems. - * - * Also the access to the ring buffer is always safe. - */ - rb_next_seq = prb_next_seq(prb); - seq = rb_next_seq - ((u32)rb_next_seq - nbcon_seq); - - return seq; -} - -#endif /* CONFIG_64BIT */ - /** * nbcon_seq_read - Read the current console sequence * @con: Console to read the sequence of @@ -183,7 +150,7 @@ u64 nbcon_seq_read(struct console *con) { unsigned long nbcon_seq = atomic_long_read(&ACCESS_PRIVATE(con, nbcon_seq)); - return __nbcon_seq_to_seq(nbcon_seq); + return __ulseq_to_u64seq(prb, nbcon_seq); } /** @@ -204,7 +171,7 @@ void nbcon_seq_force(struct console *con, u64 seq) */ u64 valid_seq = max_t(u64, seq, prb_first_valid_seq(prb)); - atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), __seq_to_nbcon_seq(valid_seq)); + atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), __u64seq_to_ulseq(valid_seq)); /* Clear con->seq since nbcon consoles use con->nbcon_seq instead. */ con->seq = 0; @@ -223,11 +190,11 @@ void nbcon_seq_force(struct console *con, u64 seq) */ static void nbcon_seq_try_update(struct nbcon_context *ctxt, u64 new_seq) { - unsigned long nbcon_seq = __seq_to_nbcon_seq(ctxt->seq); + unsigned long nbcon_seq = __u64seq_to_ulseq(ctxt->seq); struct console *con = ctxt->console; if (atomic_long_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_seq), &nbcon_seq, - __seq_to_nbcon_seq(new_seq))) { + __u64seq_to_ulseq(new_seq))) { ctxt->seq = new_seq; } else { ctxt->seq = nbcon_seq_read(con); diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 760bb70ba98a..259b8e382560 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -349,6 +349,29 @@ static bool panic_in_progress(void) return unlikely(atomic_read(&panic_cpu) != PANIC_CPU_INVALID); } +/* Return true if a panic is in progress on the current CPU. */ +bool this_cpu_in_panic(void) +{ + /* + * We can use raw_smp_processor_id() here because it is impossible for + * the task to be migrated to the panic_cpu, or away from it. If + * panic_cpu has already been set, and we're not currently executing on + * that CPU, then we never will be. + */ + return unlikely(atomic_read(&panic_cpu) == raw_smp_processor_id()); +} + +/* + * Return true if a panic is in progress on a remote CPU. + * + * On true, the local CPU should immediately release any printing resources + * that may be needed by the panic CPU. + */ +bool other_cpu_in_panic(void) +{ + return (panic_in_progress() && !this_cpu_in_panic()); +} + /* * This is used for debugging the mess that is the VT code by * keeping track if we have the console semaphore held. It's @@ -441,12 +464,6 @@ static int console_msg_format = MSG_FORMAT_DEFAULT; static DEFINE_MUTEX(syslog_lock); #ifdef CONFIG_PRINTK -/* - * During panic, heavy printk by other CPUs can delay the - * panic and risk deadlock on console resources. - */ -static int __read_mostly suppress_panic_printk; - DECLARE_WAIT_QUEUE_HEAD(log_wait); /* All 3 protected by @syslog_lock. */ /* the next printk record to read by syslog(READ) or /proc/kmsg */ @@ -1837,10 +1854,23 @@ static bool console_waiter; */ static void console_lock_spinning_enable(void) { + /* + * Do not use spinning in panic(). The panic CPU wants to keep the lock. + * Non-panic CPUs abandon the flush anyway. + * + * Just keep the lockdep annotation. The panic-CPU should avoid + * taking console_owner_lock because it might cause a deadlock. + * This looks like the easiest way how to prevent false lockdep + * reports without handling races a lockless way. + */ + if (panic_in_progress()) + goto lockdep; + raw_spin_lock(&console_owner_lock); console_owner = current; raw_spin_unlock(&console_owner_lock); +lockdep: /* The waiter may spin on us after setting console_owner */ spin_acquire(&console_owner_dep_map, 0, 0, _THIS_IP_); } @@ -1865,6 +1895,22 @@ static int console_lock_spinning_disable_and_check(int cookie) { int waiter; + /* + * Ignore spinning waiters during panic() because they might get stopped + * or blocked at any time, + * + * It is safe because nobody is allowed to start spinning during panic + * in the first place. If there has been a waiter then non panic CPUs + * might stay spinning. They would get stopped anyway. The panic context + * will never start spinning and an interrupted spin on panic CPU will + * never continue. + */ + if (panic_in_progress()) { + /* Keep lockdep happy. */ + spin_release(&console_owner_dep_map, _THIS_IP_); + return 0; + } + raw_spin_lock(&console_owner_lock); waiter = READ_ONCE(console_waiter); console_owner = NULL; @@ -2261,8 +2307,12 @@ asmlinkage int vprintk_emit(int facility, int level, if (unlikely(suppress_printk)) return 0; - if (unlikely(suppress_panic_printk) && - atomic_read(&panic_cpu) != raw_smp_processor_id()) + /* + * The messages on the panic CPU are the most important. If + * non-panic CPUs are generating any messages, they will be + * silently dropped. + */ + if (other_cpu_in_panic()) return 0; if (level == LOGLEVEL_SCHED) { @@ -2598,26 +2648,6 @@ static int console_cpu_notify(unsigned int cpu) return 0; } -/* - * Return true if a panic is in progress on a remote CPU. - * - * On true, the local CPU should immediately release any printing resources - * that may be needed by the panic CPU. - */ -bool other_cpu_in_panic(void) -{ - if (!panic_in_progress()) - return false; - - /* - * We can use raw_smp_processor_id() here because it is impossible for - * the task to be migrated to the panic_cpu, or away from it. If - * panic_cpu has already been set, and we're not currently executing on - * that CPU, then we never will be. - */ - return atomic_read(&panic_cpu) != raw_smp_processor_id(); -} - /** * console_lock - block the console subsystem from printing * @@ -2773,8 +2803,6 @@ void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped) bool printk_get_next_message(struct printk_message *pmsg, u64 seq, bool is_extended, bool may_suppress) { - static int panic_console_dropped; - struct printk_buffers *pbufs = pmsg->pbufs; const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf); const size_t outbuf_sz = sizeof(pbufs->outbuf); @@ -2802,17 +2830,6 @@ bool printk_get_next_message(struct printk_message *pmsg, u64 seq, pmsg->seq = r.info->seq; pmsg->dropped = r.info->seq - seq; - /* - * Check for dropped messages in panic here so that printk - * suppression can occur as early as possible if necessary. - */ - if (pmsg->dropped && - panic_in_progress() && - panic_console_dropped++ > 10) { - suppress_panic_printk = 1; - pr_warn_once("Too many dropped messages. Suppress messages on non-panic CPUs to prevent livelock.\n"); - } - /* Skip record that has level above the console loglevel. */ if (may_suppress && suppress_message_printing(r.info->level)) goto out; @@ -3758,7 +3775,7 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre might_sleep(); - seq = prb_next_seq(prb); + seq = prb_next_reserve_seq(prb); /* Flush the consoles so that records up to @seq are printed. */ console_lock(); diff --git a/kernel/printk/printk_ringbuffer.c b/kernel/printk/printk_ringbuffer.c index fde338606ce8..88e8f3a61922 100644 --- a/kernel/printk/printk_ringbuffer.c +++ b/kernel/printk/printk_ringbuffer.c @@ -6,6 +6,7 @@ #include #include #include "printk_ringbuffer.h" +#include "internal.h" /** * DOC: printk_ringbuffer overview @@ -303,6 +304,9 @@ * * desc_push_tail:B / desc_reserve:D * set descriptor reusable (state), then push descriptor tail (id) + * + * desc_update_last_finalized:A / desc_last_finalized_seq:A + * store finalized record, then set new highest finalized sequence number */ #define DATA_SIZE(data_ring) _DATA_SIZE((data_ring)->size_bits) @@ -1030,9 +1034,13 @@ static char *data_alloc(struct printk_ringbuffer *rb, unsigned int size, unsigned long next_lpos; if (size == 0) { - /* Specify a data-less block. */ - blk_lpos->begin = NO_LPOS; - blk_lpos->next = NO_LPOS; + /* + * Data blocks are not created for empty lines. Instead, the + * reader will recognize these special lpos values and handle + * it appropriately. + */ + blk_lpos->begin = EMPTY_LINE_LPOS; + blk_lpos->next = EMPTY_LINE_LPOS; return NULL; } @@ -1210,10 +1218,18 @@ static const char *get_data(struct prb_data_ring *data_ring, /* Data-less data block description. */ if (BLK_DATALESS(blk_lpos)) { - if (blk_lpos->begin == NO_LPOS && blk_lpos->next == NO_LPOS) { + /* + * Records that are just empty lines are also valid, even + * though they do not have a data block. For such records + * explicitly return empty string data to signify success. + */ + if (blk_lpos->begin == EMPTY_LINE_LPOS && + blk_lpos->next == EMPTY_LINE_LPOS) { *data_size = 0; return ""; } + + /* Data lost, invalid, or otherwise unavailable. */ return NULL; } @@ -1441,20 +1457,118 @@ fail_reopen: return false; } +/* + * @last_finalized_seq value guarantees that all records up to and including + * this sequence number are finalized and can be read. The only exception are + * too old records which have already been overwritten. + * + * It is also guaranteed that @last_finalized_seq only increases. + * + * Be aware that finalized records following non-finalized records are not + * reported because they are not yet available to the reader. For example, + * a new record stored via printk() will not be available to a printer if + * it follows a record that has not been finalized yet. However, once that + * non-finalized record becomes finalized, @last_finalized_seq will be + * appropriately updated and the full set of finalized records will be + * available to the printer. And since each printk() caller will either + * directly print or trigger deferred printing of all available unprinted + * records, all printk() messages will get printed. + */ +static u64 desc_last_finalized_seq(struct printk_ringbuffer *rb) +{ + struct prb_desc_ring *desc_ring = &rb->desc_ring; + unsigned long ulseq; + + /* + * Guarantee the sequence number is loaded before loading the + * associated record in order to guarantee that the record can be + * seen by this CPU. This pairs with desc_update_last_finalized:A. + */ + ulseq = atomic_long_read_acquire(&desc_ring->last_finalized_seq + ); /* LMM(desc_last_finalized_seq:A) */ + + return __ulseq_to_u64seq(rb, ulseq); +} + +static bool _prb_read_valid(struct printk_ringbuffer *rb, u64 *seq, + struct printk_record *r, unsigned int *line_count); + +/* + * Check if there are records directly following @last_finalized_seq that are + * finalized. If so, update @last_finalized_seq to the latest of these + * records. It is not allowed to skip over records that are not yet finalized. + */ +static void desc_update_last_finalized(struct printk_ringbuffer *rb) +{ + struct prb_desc_ring *desc_ring = &rb->desc_ring; + u64 old_seq = desc_last_finalized_seq(rb); + unsigned long oldval; + unsigned long newval; + u64 finalized_seq; + u64 try_seq; + +try_again: + finalized_seq = old_seq; + try_seq = finalized_seq + 1; + + /* Try to find later finalized records. */ + while (_prb_read_valid(rb, &try_seq, NULL, NULL)) { + finalized_seq = try_seq; + try_seq++; + } + + /* No update needed if no later finalized record was found. */ + if (finalized_seq == old_seq) + return; + + oldval = __u64seq_to_ulseq(old_seq); + newval = __u64seq_to_ulseq(finalized_seq); + + /* + * Set the sequence number of a later finalized record that has been + * seen. + * + * Guarantee the record data is visible to other CPUs before storing + * its sequence number. This pairs with desc_last_finalized_seq:A. + * + * Memory barrier involvement: + * + * If desc_last_finalized_seq:A reads from + * desc_update_last_finalized:A, then desc_read:A reads from + * _prb_commit:B. + * + * Relies on: + * + * RELEASE from _prb_commit:B to desc_update_last_finalized:A + * matching + * ACQUIRE from desc_last_finalized_seq:A to desc_read:A + * + * Note: _prb_commit:B and desc_update_last_finalized:A can be + * different CPUs. However, the desc_update_last_finalized:A + * CPU (which performs the release) must have previously seen + * _prb_commit:B. + */ + if (!atomic_long_try_cmpxchg_release(&desc_ring->last_finalized_seq, + &oldval, newval)) { /* LMM(desc_update_last_finalized:A) */ + old_seq = __ulseq_to_u64seq(rb, oldval); + goto try_again; + } +} + /* * Attempt to finalize a specified descriptor. If this fails, the descriptor * is either already final or it will finalize itself when the writer commits. */ -static void desc_make_final(struct prb_desc_ring *desc_ring, unsigned long id) +static void desc_make_final(struct printk_ringbuffer *rb, unsigned long id) { + struct prb_desc_ring *desc_ring = &rb->desc_ring; unsigned long prev_state_val = DESC_SV(id, desc_committed); struct prb_desc *d = to_desc(desc_ring, id); - atomic_long_cmpxchg_relaxed(&d->state_var, prev_state_val, - DESC_SV(id, desc_finalized)); /* LMM(desc_make_final:A) */ - - /* Best effort to remember the last finalized @id. */ - atomic_long_set(&desc_ring->last_finalized_id, id); + if (atomic_long_try_cmpxchg_relaxed(&d->state_var, &prev_state_val, + DESC_SV(id, desc_finalized))) { /* LMM(desc_make_final:A) */ + desc_update_last_finalized(rb); + } } /** @@ -1550,7 +1664,7 @@ bool prb_reserve(struct prb_reserved_entry *e, struct printk_ringbuffer *rb, * readers. (For seq==0 there is no previous descriptor.) */ if (info->seq > 0) - desc_make_final(desc_ring, DESC_ID(id - 1)); + desc_make_final(rb, DESC_ID(id - 1)); r->text_buf = data_alloc(rb, r->text_buf_size, &d->text_blk_lpos, id); /* If text data allocation fails, a data-less record is committed. */ @@ -1643,7 +1757,7 @@ void prb_commit(struct prb_reserved_entry *e) */ head_id = atomic_long_read(&desc_ring->head_id); /* LMM(prb_commit:A) */ if (head_id != e->id) - desc_make_final(desc_ring, e->id); + desc_make_final(e->rb, e->id); } /** @@ -1663,12 +1777,9 @@ void prb_commit(struct prb_reserved_entry *e) */ void prb_final_commit(struct prb_reserved_entry *e) { - struct prb_desc_ring *desc_ring = &e->rb->desc_ring; - _prb_commit(e, desc_finalized); - /* Best effort to remember the last finalized @id. */ - atomic_long_set(&desc_ring->last_finalized_id, e->id); + desc_update_last_finalized(e->rb); } /* @@ -1832,7 +1943,7 @@ static int prb_read(struct printk_ringbuffer *rb, u64 seq, } /* Get the sequence number of the tail descriptor. */ -static u64 prb_first_seq(struct printk_ringbuffer *rb) +u64 prb_first_seq(struct printk_ringbuffer *rb) { struct prb_desc_ring *desc_ring = &rb->desc_ring; enum desc_state d_state; @@ -1875,12 +1986,123 @@ static u64 prb_first_seq(struct printk_ringbuffer *rb) return seq; } -/* - * Non-blocking read of a record. Updates @seq to the last finalized record - * (which may have no data available). +/** + * prb_next_reserve_seq() - Get the sequence number after the most recently + * reserved record. * - * See the description of prb_read_valid() and prb_read_valid_info() - * for details. + * @rb: The ringbuffer to get the sequence number from. + * + * This is the public function available to readers to see what sequence + * number will be assigned to the next reserved record. + * + * Note that depending on the situation, this value can be equal to or + * higher than the sequence number returned by prb_next_seq(). + * + * Context: Any context. + * Return: The sequence number that will be assigned to the next record + * reserved. + */ +u64 prb_next_reserve_seq(struct printk_ringbuffer *rb) +{ + struct prb_desc_ring *desc_ring = &rb->desc_ring; + unsigned long last_finalized_id; + atomic_long_t *state_var; + u64 last_finalized_seq; + unsigned long head_id; + struct prb_desc desc; + unsigned long diff; + struct prb_desc *d; + int err; + + /* + * It may not be possible to read a sequence number for @head_id. + * So the ID of @last_finailzed_seq is used to calculate what the + * sequence number of @head_id will be. + */ + +try_again: + last_finalized_seq = desc_last_finalized_seq(rb); + + /* + * @head_id is loaded after @last_finalized_seq to ensure that + * it points to the record with @last_finalized_seq or newer. + * + * Memory barrier involvement: + * + * If desc_last_finalized_seq:A reads from + * desc_update_last_finalized:A, then + * prb_next_reserve_seq:A reads from desc_reserve:D. + * + * Relies on: + * + * RELEASE from desc_reserve:D to desc_update_last_finalized:A + * matching + * ACQUIRE from desc_last_finalized_seq:A to prb_next_reserve_seq:A + * + * Note: desc_reserve:D and desc_update_last_finalized:A can be + * different CPUs. However, the desc_update_last_finalized:A CPU + * (which performs the release) must have previously seen + * desc_read:C, which implies desc_reserve:D can be seen. + */ + head_id = atomic_long_read(&desc_ring->head_id); /* LMM(prb_next_reserve_seq:A) */ + + d = to_desc(desc_ring, last_finalized_seq); + state_var = &d->state_var; + + /* Extract the ID, used to specify the descriptor to read. */ + last_finalized_id = DESC_ID(atomic_long_read(state_var)); + + /* Ensure @last_finalized_id is correct. */ + err = desc_read_finalized_seq(desc_ring, last_finalized_id, last_finalized_seq, &desc); + + if (err == -EINVAL) { + if (last_finalized_seq == 0) { + /* + * No record has been finalized or even reserved yet. + * + * The @head_id is initialized such that the first + * increment will yield the first record (seq=0). + * Handle it separately to avoid a negative @diff + * below. + */ + if (head_id == DESC0_ID(desc_ring->count_bits)) + return 0; + + /* + * One or more descriptors are already reserved. Use + * the descriptor ID of the first one (@seq=0) for + * the @diff below. + */ + last_finalized_id = DESC0_ID(desc_ring->count_bits) + 1; + } else { + /* Record must have been overwritten. Try again. */ + goto try_again; + } + } + + /* Diff of known descriptor IDs to compute related sequence numbers. */ + diff = head_id - last_finalized_id; + + /* + * @head_id points to the most recently reserved record, but this + * function returns the sequence number that will be assigned to the + * next (not yet reserved) record. Thus +1 is needed. + */ + return (last_finalized_seq + diff + 1); +} + +/* + * Non-blocking read of a record. + * + * On success @seq is updated to the record that was read and (if provided) + * @r and @line_count will contain the read/calculated data. + * + * On failure @seq is updated to a record that is not yet available to the + * reader, but it will be the next record available to the reader. + * + * Note: When the current CPU is in panic, this function will skip over any + * non-existent/non-finalized records in order to allow the panic CPU + * to print any and all records that have been finalized. */ static bool _prb_read_valid(struct printk_ringbuffer *rb, u64 *seq, struct printk_record *r, unsigned int *line_count) @@ -1899,12 +2121,32 @@ static bool _prb_read_valid(struct printk_ringbuffer *rb, u64 *seq, *seq = tail_seq; } else if (err == -ENOENT) { - /* Record exists, but no data available. Skip. */ + /* Record exists, but the data was lost. Skip. */ (*seq)++; } else { - /* Non-existent/non-finalized record. Must stop. */ - return false; + /* + * Non-existent/non-finalized record. Must stop. + * + * For panic situations it cannot be expected that + * non-finalized records will become finalized. But + * there may be other finalized records beyond that + * need to be printed for a panic situation. If this + * is the panic CPU, skip this + * non-existent/non-finalized record unless it is + * at or beyond the head, in which case it is not + * possible to continue. + * + * Note that new messages printed on panic CPU are + * finalized when we are here. The only exception + * might be the last message without trailing newline. + * But it would have the sequence number returned + * by "prb_next_reserve_seq() - 1". + */ + if (this_cpu_in_panic() && ((*seq + 1) < prb_next_reserve_seq(rb))) + (*seq)++; + else + return false; } } @@ -1932,7 +2174,7 @@ static bool _prb_read_valid(struct printk_ringbuffer *rb, u64 *seq, * On success, the reader must check r->info.seq to see which record was * actually read. This allows the reader to detect dropped records. * - * Failure means @seq refers to a not yet written record. + * Failure means @seq refers to a record not yet available to the reader. */ bool prb_read_valid(struct printk_ringbuffer *rb, u64 seq, struct printk_record *r) @@ -1962,7 +2204,7 @@ bool prb_read_valid(struct printk_ringbuffer *rb, u64 seq, * On success, the reader must check info->seq to see which record meta data * was actually read. This allows the reader to detect dropped records. * - * Failure means @seq refers to a not yet written record. + * Failure means @seq refers to a record not yet available to the reader. */ bool prb_read_valid_info(struct printk_ringbuffer *rb, u64 seq, struct printk_info *info, unsigned int *line_count) @@ -2008,7 +2250,9 @@ u64 prb_first_valid_seq(struct printk_ringbuffer *rb) * newest sequence number available to readers will be. * * This provides readers a sequence number to jump to if all currently - * available records should be skipped. + * available records should be skipped. It is guaranteed that all records + * previous to the returned value have been finalized and are (or were) + * available to the reader. * * Context: Any context. * Return: The sequence number of the next newest (not yet available) record @@ -2016,34 +2260,19 @@ u64 prb_first_valid_seq(struct printk_ringbuffer *rb) */ u64 prb_next_seq(struct printk_ringbuffer *rb) { - struct prb_desc_ring *desc_ring = &rb->desc_ring; - enum desc_state d_state; - unsigned long id; u64 seq; - /* Check if the cached @id still points to a valid @seq. */ - id = atomic_long_read(&desc_ring->last_finalized_id); - d_state = desc_read(desc_ring, id, NULL, &seq, NULL); + seq = desc_last_finalized_seq(rb); - if (d_state == desc_finalized || d_state == desc_reusable) { - /* - * Begin searching after the last finalized record. - * - * On 0, the search must begin at 0 because of hack#2 - * of the bootstrapping phase it is not known if a - * record at index 0 exists. - */ - if (seq != 0) - seq++; - } else { - /* - * The information about the last finalized sequence number - * has gone. It should happen only when there is a flood of - * new messages and the ringbuffer is rapidly recycled. - * Give up and start from the beginning. - */ - seq = 0; - } + /* + * Begin searching after the last finalized record. + * + * On 0, the search must begin at 0 because of hack#2 + * of the bootstrapping phase it is not known if a + * record at index 0 exists. + */ + if (seq != 0) + seq++; /* * The information about the last finalized @seq might be inaccurate. @@ -2085,7 +2314,7 @@ void prb_init(struct printk_ringbuffer *rb, rb->desc_ring.infos = infos; atomic_long_set(&rb->desc_ring.head_id, DESC0_ID(descbits)); atomic_long_set(&rb->desc_ring.tail_id, DESC0_ID(descbits)); - atomic_long_set(&rb->desc_ring.last_finalized_id, DESC0_ID(descbits)); + atomic_long_set(&rb->desc_ring.last_finalized_seq, 0); rb->text_data_ring.size_bits = textbits; rb->text_data_ring.data = text_buf; diff --git a/kernel/printk/printk_ringbuffer.h b/kernel/printk/printk_ringbuffer.h index 18cd25e489b8..52626d0f1fa3 100644 --- a/kernel/printk/printk_ringbuffer.h +++ b/kernel/printk/printk_ringbuffer.h @@ -75,7 +75,7 @@ struct prb_desc_ring { struct printk_info *infos; atomic_long_t head_id; atomic_long_t tail_id; - atomic_long_t last_finalized_id; + atomic_long_t last_finalized_seq; }; /* @@ -127,8 +127,22 @@ enum desc_state { #define DESC_SV(id, state) (((unsigned long)state << DESC_FLAGS_SHIFT) | id) #define DESC_ID_MASK (~DESC_FLAGS_MASK) #define DESC_ID(sv) ((sv) & DESC_ID_MASK) + +/* + * Special data block logical position values (for fields of + * @prb_desc.text_blk_lpos). + * + * - Bit0 is used to identify if the record has no data block. (Implemented in + * the LPOS_DATALESS() macro.) + * + * - Bit1 specifies the reason for not having a data block. + * + * These special values could never be real lpos values because of the + * meta data and alignment padding of data blocks. (See to_blk_size() for + * details.) + */ #define FAILED_LPOS 0x1 -#define NO_LPOS 0x3 +#define EMPTY_LINE_LPOS 0x3 #define FAILED_BLK_LPOS \ { \ @@ -259,7 +273,7 @@ static struct printk_ringbuffer name = { \ .infos = &_##name##_infos[0], \ .head_id = ATOMIC_INIT(DESC0_ID(descbits)), \ .tail_id = ATOMIC_INIT(DESC0_ID(descbits)), \ - .last_finalized_id = ATOMIC_INIT(DESC0_ID(descbits)), \ + .last_finalized_seq = ATOMIC_INIT(0), \ }, \ .text_data_ring = { \ .size_bits = (avgtextbits) + (descbits), \ @@ -378,7 +392,41 @@ bool prb_read_valid(struct printk_ringbuffer *rb, u64 seq, bool prb_read_valid_info(struct printk_ringbuffer *rb, u64 seq, struct printk_info *info, unsigned int *line_count); +u64 prb_first_seq(struct printk_ringbuffer *rb); u64 prb_first_valid_seq(struct printk_ringbuffer *rb); u64 prb_next_seq(struct printk_ringbuffer *rb); +u64 prb_next_reserve_seq(struct printk_ringbuffer *rb); + +#ifdef CONFIG_64BIT + +#define __u64seq_to_ulseq(u64seq) (u64seq) +#define __ulseq_to_u64seq(rb, ulseq) (ulseq) + +#else /* CONFIG_64BIT */ + +#define __u64seq_to_ulseq(u64seq) ((u32)u64seq) + +static inline u64 __ulseq_to_u64seq(struct printk_ringbuffer *rb, u32 ulseq) +{ + u64 rb_first_seq = prb_first_seq(rb); + u64 seq; + + /* + * The provided sequence is only the lower 32 bits of the ringbuffer + * sequence. It needs to be expanded to 64bit. Get the first sequence + * number from the ringbuffer and fold it. + * + * Having a 32bit representation in the console is sufficient. + * If a console ever gets more than 2^31 records behind + * the ringbuffer then this is the least of the problems. + * + * Also the access to the ring buffer is always safe. + */ + seq = rb_first_seq - (s32)((u32)rb_first_seq - ulseq); + + return seq; +} + +#endif /* CONFIG_64BIT */ #endif /* _KERNEL_PRINTK_RINGBUFFER_H */ diff --git a/lib/dump_stack.c b/lib/dump_stack.c index 83471e81501a..222c6d6c8281 100644 --- a/lib/dump_stack.c +++ b/lib/dump_stack.c @@ -96,15 +96,25 @@ static void __dump_stack(const char *log_lvl) */ asmlinkage __visible void dump_stack_lvl(const char *log_lvl) { + bool in_panic = this_cpu_in_panic(); unsigned long flags; /* * Permit this cpu to perform nested stack dumps while serialising - * against other CPUs + * against other CPUs, unless this CPU is in panic. + * + * When in panic, non-panic CPUs are not permitted to store new + * printk messages so there is no need to synchronize the output. + * This avoids potential deadlock in panic() if another CPU is + * holding and unable to release the printk_cpu_sync. */ - printk_cpu_sync_get_irqsave(flags); + if (!in_panic) + printk_cpu_sync_get_irqsave(flags); + __dump_stack(log_lvl); - printk_cpu_sync_put_irqrestore(flags); + + if (!in_panic) + printk_cpu_sync_put_irqrestore(flags); } EXPORT_SYMBOL(dump_stack_lvl); diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug index 321ab379994f..afc72fde0f03 100644 --- a/mm/Kconfig.debug +++ b/mm/Kconfig.debug @@ -64,11 +64,11 @@ config SLUB_DEBUG_ON help Boot with debugging on by default. SLUB boots by default with the runtime debug capabilities switched off. Enabling this is - equivalent to specifying the "slub_debug" parameter on boot. + equivalent to specifying the "slab_debug" parameter on boot. There is no support for more fine grained debug control like - possible with slub_debug=xxx. SLUB debugging may be switched + possible with slab_debug=xxx. SLUB debugging may be switched off in a kernel built with CONFIG_SLUB_DEBUG_ON by specifying - "slub_debug=-". + "slab_debug=-". config PAGE_OWNER bool "Track page owner" diff --git a/mm/kasan/generic.c b/mm/kasan/generic.c index 1900f8576034..6310a180278b 100644 --- a/mm/kasan/generic.c +++ b/mm/kasan/generic.c @@ -334,14 +334,6 @@ DEFINE_ASAN_SET_SHADOW(f3); DEFINE_ASAN_SET_SHADOW(f5); DEFINE_ASAN_SET_SHADOW(f8); -/* Only allow cache merging when no per-object metadata is present. */ -slab_flags_t kasan_never_merge(void) -{ - if (!kasan_requires_meta()) - return 0; - return SLAB_KASAN; -} - /* * Adaptive redzone policy taken from the userspace AddressSanitizer runtime. * For larger allocations larger redzones are used. @@ -370,15 +362,13 @@ void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, return; /* - * SLAB_KASAN is used to mark caches that are sanitized by KASAN - * and that thus have per-object metadata. - * Currently this flag is used in two places: - * 1. In slab_ksize() to account for per-object metadata when - * calculating the size of the accessible memory within the object. - * 2. In slab_common.c via kasan_never_merge() to prevent merging of - * caches with per-object metadata. + * SLAB_KASAN is used to mark caches that are sanitized by KASAN and + * that thus have per-object metadata. Currently, this flag is used in + * slab_ksize() to account for per-object metadata when calculating the + * size of the accessible memory within the object. Additionally, we use + * SLAB_NO_MERGE to prevent merging of caches with per-object metadata. */ - *flags |= SLAB_KASAN; + *flags |= SLAB_KASAN | SLAB_NO_MERGE; ok_size = *size; diff --git a/mm/slab.h b/mm/slab.h index 54deeb0428c6..d2bc9b191222 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -363,7 +363,6 @@ static inline int objs_per_slab(const struct kmem_cache *cache, enum slab_state { DOWN, /* No slab functionality yet */ PARTIAL, /* SLUB: kmem_cache_node available */ - PARTIAL_NODE, /* SLAB: kmalloc size for node struct available */ UP, /* Slab caches usable but not all extras yet */ FULL /* Everything is working */ }; @@ -387,7 +386,7 @@ extern const struct kmalloc_info_struct { /* Kmalloc array related functions */ void setup_kmalloc_cache_index_table(void); -void create_kmalloc_caches(slab_flags_t); +void create_kmalloc_caches(void); extern u8 kmalloc_size_index[24]; @@ -422,8 +421,6 @@ gfp_t kmalloc_fix_flags(gfp_t flags); int __kmem_cache_create(struct kmem_cache *, slab_flags_t flags); void __init kmem_cache_init(void); -void __init new_kmalloc_cache(int idx, enum kmalloc_cache_type type, - slab_flags_t flags); extern void create_boot_cache(struct kmem_cache *, const char *name, unsigned int size, slab_flags_t flags, unsigned int useroffset, unsigned int usersize); @@ -435,8 +432,7 @@ struct kmem_cache * __kmem_cache_alias(const char *name, unsigned int size, unsigned int align, slab_flags_t flags, void (*ctor)(void *)); -slab_flags_t kmem_cache_flags(unsigned int object_size, - slab_flags_t flags, const char *name); +slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name); static inline bool is_kmalloc_cache(struct kmem_cache *s) { @@ -469,7 +465,6 @@ static inline bool is_kmalloc_cache(struct kmem_cache *s) SLAB_STORE_USER | \ SLAB_TRACE | \ SLAB_CONSISTENCY_CHECKS | \ - SLAB_MEM_SPREAD | \ SLAB_NOLEAKTRACE | \ SLAB_RECLAIM_ACCOUNT | \ SLAB_TEMPORARY | \ @@ -528,7 +523,7 @@ static inline bool __slub_debug_enabled(void) #endif /* - * Returns true if any of the specified slub_debug flags is enabled for the + * Returns true if any of the specified slab_debug flags is enabled for the * cache. Use only for flags parsed by setup_slub_debug() as it also enables * the static key. */ diff --git a/mm/slab_common.c b/mm/slab_common.c index 238293b1dbe1..23af762148ca 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -50,7 +50,7 @@ static DECLARE_WORK(slab_caches_to_rcu_destroy_work, */ #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ SLAB_TRACE | SLAB_TYPESAFE_BY_RCU | SLAB_NOLEAKTRACE | \ - SLAB_FAILSLAB | SLAB_NO_MERGE | kasan_never_merge()) + SLAB_FAILSLAB | SLAB_NO_MERGE) #define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \ SLAB_CACHE_DMA32 | SLAB_ACCOUNT) @@ -172,7 +172,7 @@ struct kmem_cache *find_mergeable(unsigned int size, unsigned int align, size = ALIGN(size, sizeof(void *)); align = calculate_alignment(flags, align, size); size = ALIGN(size, align); - flags = kmem_cache_flags(size, flags, name); + flags = kmem_cache_flags(flags, name); if (flags & SLAB_NEVER_MERGE) return NULL; @@ -282,7 +282,7 @@ kmem_cache_create_usercopy(const char *name, #ifdef CONFIG_SLUB_DEBUG /* - * If no slub_debug was enabled globally, the static key is not yet + * If no slab_debug was enabled globally, the static key is not yet * enabled by setup_slub_debug(). Enable it if the cache is being * created with any of the debugging flags passed explicitly. * It's also possible that this is the first cache created with @@ -404,8 +404,12 @@ EXPORT_SYMBOL(kmem_cache_create); */ static void kmem_cache_release(struct kmem_cache *s) { - sysfs_slab_unlink(s); - sysfs_slab_release(s); + if (slab_state >= FULL) { + sysfs_slab_unlink(s); + sysfs_slab_release(s); + } else { + slab_kmem_cache_release(s); + } } #else static void kmem_cache_release(struct kmem_cache *s) @@ -766,7 +770,7 @@ EXPORT_SYMBOL(kmalloc_size_roundup); } /* - * kmalloc_info[] is to make slub_debug=,kmalloc-xx option work at boot time. + * kmalloc_info[] is to make slab_debug=,kmalloc-xx option work at boot time. * kmalloc_index() supports up to 2^21=2MB, so the final entry of the table is * kmalloc-2M. */ @@ -853,9 +857,10 @@ static unsigned int __kmalloc_minalign(void) return max(minalign, arch_slab_minalign()); } -void __init -new_kmalloc_cache(int idx, enum kmalloc_cache_type type, slab_flags_t flags) +static void __init +new_kmalloc_cache(int idx, enum kmalloc_cache_type type) { + slab_flags_t flags = 0; unsigned int minalign = __kmalloc_minalign(); unsigned int aligned_size = kmalloc_info[idx].size; int aligned_idx = idx; @@ -902,7 +907,7 @@ new_kmalloc_cache(int idx, enum kmalloc_cache_type type, slab_flags_t flags) * may already have been created because they were needed to * enable allocations for slab creation. */ -void __init create_kmalloc_caches(slab_flags_t flags) +void __init create_kmalloc_caches(void) { int i; enum kmalloc_cache_type type; @@ -913,7 +918,7 @@ void __init create_kmalloc_caches(slab_flags_t flags) for (type = KMALLOC_NORMAL; type < NR_KMALLOC_TYPES; type++) { for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) { if (!kmalloc_caches[type][i]) - new_kmalloc_cache(i, type, flags); + new_kmalloc_cache(i, type); /* * Caches that are not of the two-to-the-power-of size. @@ -922,10 +927,10 @@ void __init create_kmalloc_caches(slab_flags_t flags) */ if (KMALLOC_MIN_SIZE <= 32 && i == 6 && !kmalloc_caches[type][1]) - new_kmalloc_cache(1, type, flags); + new_kmalloc_cache(1, type); if (KMALLOC_MIN_SIZE <= 64 && i == 7 && !kmalloc_caches[type][2]) - new_kmalloc_cache(2, type, flags); + new_kmalloc_cache(2, type); } } #ifdef CONFIG_RANDOM_KMALLOC_CACHES diff --git a/mm/slub.c b/mm/slub.c index 2ef88bbf56a3..1bb2a93cf7b6 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -295,7 +295,7 @@ static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) /* * Debugging flags that require metadata to be stored in the slab. These get - * disabled when slub_debug=O is used and a cache's min order increases with + * disabled when slab_debug=O is used and a cache's min order increases with * metadata. */ #define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER) @@ -306,13 +306,13 @@ static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) /* Internal SLUB flags */ /* Poison object */ -#define __OBJECT_POISON ((slab_flags_t __force)0x80000000U) +#define __OBJECT_POISON __SLAB_FLAG_BIT(_SLAB_OBJECT_POISON) /* Use cmpxchg_double */ #ifdef system_has_freelist_aba -#define __CMPXCHG_DOUBLE ((slab_flags_t __force)0x40000000U) +#define __CMPXCHG_DOUBLE __SLAB_FLAG_BIT(_SLAB_CMPXCHG_DOUBLE) #else -#define __CMPXCHG_DOUBLE ((slab_flags_t __force)0U) +#define __CMPXCHG_DOUBLE __SLAB_FLAG_UNUSED #endif /* @@ -391,7 +391,7 @@ struct kmem_cache_cpu { }; struct slab *slab; /* The slab from which we are allocating */ #ifdef CONFIG_SLUB_CPU_PARTIAL - struct slab *partial; /* Partially allocated frozen slabs */ + struct slab *partial; /* Partially allocated slabs */ #endif local_lock_t lock; /* Protects the fields above */ #ifdef CONFIG_SLUB_STATS @@ -1498,16 +1498,8 @@ static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects) { struct kmem_cache_node *n = get_node(s, node); - /* - * May be called early in order to allocate a slab for the - * kmem_cache_node structure. Solve the chicken-egg - * dilemma by deferring the increment of the count during - * bootstrap (see early_kmem_cache_node_alloc). - */ - if (likely(n)) { - atomic_long_inc(&n->nr_slabs); - atomic_long_add(objects, &n->total_objects); - } + atomic_long_inc(&n->nr_slabs); + atomic_long_add(objects, &n->total_objects); } static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects) { @@ -1616,7 +1608,7 @@ static inline int free_consistency_checks(struct kmem_cache *s, } /* - * Parse a block of slub_debug options. Blocks are delimited by ';' + * Parse a block of slab_debug options. Blocks are delimited by ';' * * @str: start of block * @flags: returns parsed flags, or DEBUG_DEFAULT_FLAGS if none specified @@ -1677,7 +1669,7 @@ parse_slub_debug_flags(char *str, slab_flags_t *flags, char **slabs, bool init) break; default: if (init) - pr_err("slub_debug option '%c' unknown. skipped\n", *str); + pr_err("slab_debug option '%c' unknown. skipped\n", *str); } } check_slabs: @@ -1736,7 +1728,7 @@ static int __init setup_slub_debug(char *str) /* * For backwards compatibility, a single list of flags with list of * slabs means debugging is only changed for those slabs, so the global - * slub_debug should be unchanged (0 or DEBUG_DEFAULT_FLAGS, depending + * slab_debug should be unchanged (0 or DEBUG_DEFAULT_FLAGS, depending * on CONFIG_SLUB_DEBUG_ON). We can extended that to multiple lists as * long as there is no option specifying flags without a slab list. */ @@ -1760,21 +1752,20 @@ out: return 1; } -__setup("slub_debug", setup_slub_debug); +__setup("slab_debug", setup_slub_debug); +__setup_param("slub_debug", slub_debug, setup_slub_debug, 0); /* * kmem_cache_flags - apply debugging options to the cache - * @object_size: the size of an object without meta data * @flags: flags to set * @name: name of the cache * * Debug option(s) are applied to @flags. In addition to the debug * option(s), if a slab name (or multiple) is specified i.e. - * slub_debug=,, ... + * slab_debug=,, ... * then only the select slabs will receive the debug option(s). */ -slab_flags_t kmem_cache_flags(unsigned int object_size, - slab_flags_t flags, const char *name) +slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name) { char *iter; size_t len; @@ -1850,8 +1841,7 @@ static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab) {} static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab) {} -slab_flags_t kmem_cache_flags(unsigned int object_size, - slab_flags_t flags, const char *name) +slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name) { return flags; } @@ -2038,11 +2028,6 @@ void memcg_slab_alloc_error_hook(struct kmem_cache *s, int objects, obj_cgroup_uncharge(objcg, objects * obj_full_size(s)); } #else /* CONFIG_MEMCG_KMEM */ -static inline struct mem_cgroup *memcg_from_slab_obj(void *ptr) -{ - return NULL; -} - static inline void memcg_free_slab_cgroups(struct slab *slab) { } @@ -2243,7 +2228,7 @@ static void __init init_freelist_randomization(void) } /* Get the next entry on the pre-computed freelist randomized */ -static void *next_freelist_entry(struct kmem_cache *s, struct slab *slab, +static void *next_freelist_entry(struct kmem_cache *s, unsigned long *pos, void *start, unsigned long page_limit, unsigned long freelist_count) @@ -2282,13 +2267,12 @@ static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) start = fixup_red_left(s, slab_address(slab)); /* First entry is used as the base of the freelist */ - cur = next_freelist_entry(s, slab, &pos, start, page_limit, - freelist_count); + cur = next_freelist_entry(s, &pos, start, page_limit, freelist_count); cur = setup_object(s, cur); slab->freelist = cur; for (idx = 1; idx < slab->objects; idx++) { - next = next_freelist_entry(s, slab, &pos, start, page_limit, + next = next_freelist_entry(s, &pos, start, page_limit, freelist_count); next = setup_object(s, next); set_freepointer(s, cur, next); @@ -3263,7 +3247,7 @@ slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) oo_order(s->min)); if (oo_order(s->min) > get_order(s->object_size)) - pr_warn(" %s debugging increased min order, use slub_debug=O to disable.\n", + pr_warn(" %s debugging increased min order, use slab_debug=O to disable.\n", s->name); for_each_kmem_cache_node(s, node, n) { @@ -3326,7 +3310,6 @@ static inline void *get_freelist(struct kmem_cache *s, struct slab *slab) counters = slab->counters; new.counters = counters; - VM_BUG_ON(!new.frozen); new.inuse = slab->objects; new.frozen = freelist != NULL; @@ -3498,18 +3481,20 @@ new_slab: slab = slub_percpu_partial(c); slub_set_percpu_partial(c, slab); - local_unlock_irqrestore(&s->cpu_slab->lock, flags); - stat(s, CPU_PARTIAL_ALLOC); - if (unlikely(!node_match(slab, node) || - !pfmemalloc_match(slab, gfpflags))) { - slab->next = NULL; - __put_partials(s, slab); - continue; + if (likely(node_match(slab, node) && + pfmemalloc_match(slab, gfpflags))) { + c->slab = slab; + freelist = get_freelist(s, slab); + VM_BUG_ON(!freelist); + stat(s, CPU_PARTIAL_ALLOC); + goto load_freelist; } - freelist = freeze_slab(s, slab); - goto retry_load_slab; + local_unlock_irqrestore(&s->cpu_slab->lock, flags); + + slab->next = NULL; + __put_partials(s, slab); } #endif @@ -3792,11 +3777,11 @@ void slab_post_alloc_hook(struct kmem_cache *s, struct obj_cgroup *objcg, zero_size = orig_size; /* - * When slub_debug is enabled, avoid memory initialization integrated + * When slab_debug is enabled, avoid memory initialization integrated * into KASAN and instead zero out the memory via the memset below with * the proper size. Otherwise, KASAN might overwrite SLUB redzones and * cause false-positive reports. This does not lead to a performance - * penalty on production builds, as slub_debug is not intended to be + * penalty on production builds, as slab_debug is not intended to be * enabled there. */ if (__slub_debug_enabled()) @@ -4187,7 +4172,6 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab, * then add it. */ if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) { - remove_full(s, n, slab); add_partial(n, slab, DEACTIVATE_TO_TAIL); stat(s, FREE_ADD_PARTIAL); } @@ -4201,9 +4185,6 @@ slab_empty: */ remove_partial(n, slab); stat(s, FREE_REMOVE_PARTIAL); - } else { - /* Slab must be on the full list */ - remove_full(s, n, slab); } spin_unlock_irqrestore(&n->list_lock, flags); @@ -4702,8 +4683,8 @@ static unsigned int slub_min_objects; * activity on the partial lists which requires taking the list_lock. This is * less a concern for large slabs though which are rarely used. * - * slub_max_order specifies the order where we begin to stop considering the - * number of objects in a slab as critical. If we reach slub_max_order then + * slab_max_order specifies the order where we begin to stop considering the + * number of objects in a slab as critical. If we reach slab_max_order then * we try to keep the page order as low as possible. So we accept more waste * of space in favor of a small page order. * @@ -4770,14 +4751,14 @@ static inline int calculate_order(unsigned int size) * and backing off gradually. * * We start with accepting at most 1/16 waste and try to find the - * smallest order from min_objects-derived/slub_min_order up to - * slub_max_order that will satisfy the constraint. Note that increasing + * smallest order from min_objects-derived/slab_min_order up to + * slab_max_order that will satisfy the constraint. Note that increasing * the order can only result in same or less fractional waste, not more. * * If that fails, we increase the acceptable fraction of waste and try * again. The last iteration with fraction of 1/2 would effectively * accept any waste and give us the order determined by min_objects, as - * long as at least single object fits within slub_max_order. + * long as at least single object fits within slab_max_order. */ for (unsigned int fraction = 16; fraction > 1; fraction /= 2) { order = calc_slab_order(size, min_order, slub_max_order, @@ -4787,7 +4768,7 @@ static inline int calculate_order(unsigned int size) } /* - * Doh this slab cannot be placed using slub_max_order. + * Doh this slab cannot be placed using slab_max_order. */ order = get_order(size); if (order <= MAX_PAGE_ORDER) @@ -4857,7 +4838,6 @@ static void early_kmem_cache_node_alloc(int node) slab = new_slab(kmem_cache_node, GFP_NOWAIT, node); BUG_ON(!slab); - inc_slabs_node(kmem_cache_node, slab_nid(slab), slab->objects); if (slab_nid(slab) != node) { pr_err("SLUB: Unable to allocate memory from node %d\n", node); pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n"); @@ -5104,7 +5084,7 @@ static int calculate_sizes(struct kmem_cache *s) static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags) { - s->flags = kmem_cache_flags(s->size, flags, s->name); + s->flags = kmem_cache_flags(flags, s->name); #ifdef CONFIG_SLAB_FREELIST_HARDENED s->random = get_random_long(); #endif @@ -5313,7 +5293,9 @@ static int __init setup_slub_min_order(char *str) return 1; } -__setup("slub_min_order=", setup_slub_min_order); +__setup("slab_min_order=", setup_slub_min_order); +__setup_param("slub_min_order=", slub_min_order, setup_slub_min_order, 0); + static int __init setup_slub_max_order(char *str) { @@ -5326,7 +5308,8 @@ static int __init setup_slub_max_order(char *str) return 1; } -__setup("slub_max_order=", setup_slub_max_order); +__setup("slab_max_order=", setup_slub_max_order); +__setup_param("slub_max_order=", slub_max_order, setup_slub_max_order, 0); static int __init setup_slub_min_objects(char *str) { @@ -5335,7 +5318,8 @@ static int __init setup_slub_min_objects(char *str) return 1; } -__setup("slub_min_objects=", setup_slub_min_objects); +__setup("slab_min_objects=", setup_slub_min_objects); +__setup_param("slub_min_objects=", slub_min_objects, setup_slub_min_objects, 0); #ifdef CONFIG_HARDENED_USERCOPY /* @@ -5663,7 +5647,7 @@ void __init kmem_cache_init(void) /* Now we can use the kmem_cache to allocate kmalloc slabs */ setup_kmalloc_cache_index_table(); - create_kmalloc_caches(0); + create_kmalloc_caches(); /* Setup random freelists for each cache */ init_freelist_randomization(); @@ -6792,14 +6776,12 @@ out_del_kobj: void sysfs_slab_unlink(struct kmem_cache *s) { - if (slab_state >= FULL) - kobject_del(&s->kobj); + kobject_del(&s->kobj); } void sysfs_slab_release(struct kmem_cache *s) { - if (slab_state >= FULL) - kobject_put(&s->kobj); + kobject_put(&s->kobj); } /* diff --git a/net/sunrpc/rpc_pipe.c b/net/sunrpc/rpc_pipe.c index dcc2b4f49e77..910a5d850d04 100644 --- a/net/sunrpc/rpc_pipe.c +++ b/net/sunrpc/rpc_pipe.c @@ -1490,7 +1490,7 @@ int register_rpc_pipefs(void) rpc_inode_cachep = kmem_cache_create("rpc_inode_cache", sizeof(struct rpc_inode), 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| - SLAB_MEM_SPREAD|SLAB_ACCOUNT), + SLAB_ACCOUNT), init_once); if (!rpc_inode_cachep) return -ENOMEM; diff --git a/scripts/mod/devicetable-offsets.c b/scripts/mod/devicetable-offsets.c index e91a3c38143b..518200813d4e 100644 --- a/scripts/mod/devicetable-offsets.c +++ b/scripts/mod/devicetable-offsets.c @@ -274,5 +274,8 @@ int main(void) DEVID(vchiq_device_id); DEVID_FIELD(vchiq_device_id, name); + DEVID(coreboot_device_id); + DEVID_FIELD(coreboot_device_id, tag); + return 0; } diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c index 4829680a0a6d..5d1c61fa5a55 100644 --- a/scripts/mod/file2alias.c +++ b/scripts/mod/file2alias.c @@ -1494,6 +1494,15 @@ static int do_vchiq_entry(const char *filename, void *symval, char *alias) return 1; } +/* Looks like: coreboot:tN */ +static int do_coreboot_entry(const char *filename, void *symval, char *alias) +{ + DEF_FIELD(symval, coreboot_device_id, tag); + sprintf(alias, "coreboot:t%08X", tag); + + return 1; +} + /* Does namelen bytes of name exactly match the symbol? */ static bool sym_is(const char *name, unsigned namelen, const char *symbol) { @@ -1575,6 +1584,7 @@ static const struct devtable devtable[] = { {"ishtp", SIZE_ishtp_device_id, do_ishtp_entry}, {"cdx", SIZE_cdx_device_id, do_cdx_entry}, {"vchiq", SIZE_vchiq_device_id, do_vchiq_entry}, + {"coreboot", SIZE_coreboot_device_id, do_coreboot_entry}, }; /* Create MODULE_ALIAS() statements. diff --git a/security/integrity/Makefile b/security/integrity/Makefile index d0ffe37dc1d6..92b63039c654 100644 --- a/security/integrity/Makefile +++ b/security/integrity/Makefile @@ -18,5 +18,6 @@ integrity-$(CONFIG_LOAD_IPL_KEYS) += platform_certs/load_ipl_s390.o integrity-$(CONFIG_LOAD_PPC_KEYS) += platform_certs/efi_parser.o \ platform_certs/load_powerpc.o \ platform_certs/keyring_handler.o +# The relative order of the 'ima' and 'evm' LSMs depends on the order below. obj-$(CONFIG_IMA) += ima/ obj-$(CONFIG_EVM) += evm/ diff --git a/security/integrity/digsig_asymmetric.c b/security/integrity/digsig_asymmetric.c index 895f4b9ce8c6..de603cf42ac7 100644 --- a/security/integrity/digsig_asymmetric.c +++ b/security/integrity/digsig_asymmetric.c @@ -132,26 +132,3 @@ out: pr_debug("%s() = %d\n", __func__, ret); return ret; } - -/** - * integrity_kernel_module_request - prevent crypto-pkcs1pad(rsa,*) requests - * @kmod_name: kernel module name - * - * We have situation, when public_key_verify_signature() in case of RSA - * algorithm use alg_name to store internal information in order to - * construct an algorithm on the fly, but crypto_larval_lookup() will try - * to use alg_name in order to load kernel module with same name. - * Since we don't have any real "crypto-pkcs1pad(rsa,*)" kernel modules, - * we are safe to fail such module request from crypto_larval_lookup(). - * - * In this way we prevent modprobe execution during digsig verification - * and avoid possible deadlock if modprobe and/or it's dependencies - * also signed with digsig. - */ -int integrity_kernel_module_request(char *kmod_name) -{ - if (strncmp(kmod_name, "crypto-pkcs1pad(rsa,", 20) == 0) - return -EINVAL; - - return 0; -} diff --git a/security/integrity/evm/Kconfig b/security/integrity/evm/Kconfig index fba9ee359bc9..861b3bacab82 100644 --- a/security/integrity/evm/Kconfig +++ b/security/integrity/evm/Kconfig @@ -6,6 +6,7 @@ config EVM select CRYPTO_HMAC select CRYPTO_SHA1 select CRYPTO_HASH_INFO + select SECURITY_PATH default n help EVM protects a file's security extended attributes against diff --git a/security/integrity/evm/evm.h b/security/integrity/evm/evm.h index 53bd7fec93fa..eb1a2c343bd7 100644 --- a/security/integrity/evm/evm.h +++ b/security/integrity/evm/evm.h @@ -32,6 +32,25 @@ struct xattr_list { bool enabled; }; +#define EVM_NEW_FILE 0x00000001 +#define EVM_IMMUTABLE_DIGSIG 0x00000002 + +/* EVM integrity metadata associated with an inode */ +struct evm_iint_cache { + unsigned long flags; + enum integrity_status evm_status:4; +}; + +extern struct lsm_blob_sizes evm_blob_sizes; + +static inline struct evm_iint_cache *evm_iint_inode(const struct inode *inode) +{ + if (unlikely(!inode->i_security)) + return NULL; + + return inode->i_security + evm_blob_sizes.lbs_inode; +} + extern int evm_initialized; #define EVM_ATTR_FSUUID 0x0001 diff --git a/security/integrity/evm/evm_crypto.c b/security/integrity/evm/evm_crypto.c index b1ffd4cc0b44..7552d49d0725 100644 --- a/security/integrity/evm/evm_crypto.c +++ b/security/integrity/evm/evm_crypto.c @@ -322,10 +322,10 @@ int evm_calc_hash(struct dentry *dentry, const char *req_xattr_name, static int evm_is_immutable(struct dentry *dentry, struct inode *inode) { const struct evm_ima_xattr_data *xattr_data = NULL; - struct integrity_iint_cache *iint; + struct evm_iint_cache *iint; int rc = 0; - iint = integrity_iint_find(inode); + iint = evm_iint_inode(inode); if (iint && (iint->flags & EVM_IMMUTABLE_DIGSIG)) return 1; diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c index cc7956d7878b..81dbade5b9b3 100644 --- a/security/integrity/evm/evm_main.c +++ b/security/integrity/evm/evm_main.c @@ -178,14 +178,14 @@ static int is_unsupported_fs(struct dentry *dentry) static enum integrity_status evm_verify_hmac(struct dentry *dentry, const char *xattr_name, char *xattr_value, - size_t xattr_value_len, - struct integrity_iint_cache *iint) + size_t xattr_value_len) { struct evm_ima_xattr_data *xattr_data = NULL; struct signature_v2_hdr *hdr; enum integrity_status evm_status = INTEGRITY_PASS; struct evm_digest digest; - struct inode *inode; + struct inode *inode = d_backing_inode(dentry); + struct evm_iint_cache *iint = evm_iint_inode(inode); int rc, xattr_len, evm_immutable = 0; if (iint && (iint->evm_status == INTEGRITY_PASS || @@ -254,8 +254,6 @@ static enum integrity_status evm_verify_hmac(struct dentry *dentry, (const char *)xattr_data, xattr_len, digest.digest, digest.hdr.length); if (!rc) { - inode = d_backing_inode(dentry); - if (xattr_data->type == EVM_XATTR_PORTABLE_DIGSIG) { if (iint) iint->flags |= EVM_IMMUTABLE_DIGSIG; @@ -403,7 +401,6 @@ int evm_read_protected_xattrs(struct dentry *dentry, u8 *buffer, * @xattr_name: requested xattr * @xattr_value: requested xattr value * @xattr_value_len: requested xattr value length - * @iint: inode integrity metadata * * Calculate the HMAC for the given dentry and verify it against the stored * security.evm xattr. For performance, use the xattr value and length @@ -416,8 +413,7 @@ int evm_read_protected_xattrs(struct dentry *dentry, u8 *buffer, */ enum integrity_status evm_verifyxattr(struct dentry *dentry, const char *xattr_name, - void *xattr_value, size_t xattr_value_len, - struct integrity_iint_cache *iint) + void *xattr_value, size_t xattr_value_len) { if (!evm_key_loaded() || !evm_protected_xattr(xattr_name)) return INTEGRITY_UNKNOWN; @@ -425,13 +421,8 @@ enum integrity_status evm_verifyxattr(struct dentry *dentry, if (is_unsupported_fs(dentry)) return INTEGRITY_UNKNOWN; - if (!iint) { - iint = integrity_iint_find(d_backing_inode(dentry)); - if (!iint) - return INTEGRITY_UNKNOWN; - } return evm_verify_hmac(dentry, xattr_name, xattr_value, - xattr_value_len, iint); + xattr_value_len); } EXPORT_SYMBOL_GPL(evm_verifyxattr); @@ -448,7 +439,7 @@ static enum integrity_status evm_verify_current_integrity(struct dentry *dentry) if (!evm_key_loaded() || !S_ISREG(inode->i_mode) || evm_fixmode) return INTEGRITY_PASS; - return evm_verify_hmac(dentry, NULL, NULL, 0, NULL); + return evm_verify_hmac(dentry, NULL, NULL, 0); } /* @@ -526,14 +517,14 @@ static int evm_protect_xattr(struct mnt_idmap *idmap, evm_status = evm_verify_current_integrity(dentry); if (evm_status == INTEGRITY_NOXATTRS) { - struct integrity_iint_cache *iint; + struct evm_iint_cache *iint; /* Exception if the HMAC is not going to be calculated. */ if (evm_hmac_disabled()) return 0; - iint = integrity_iint_find(d_backing_inode(dentry)); - if (iint && (iint->flags & IMA_NEW_FILE)) + iint = evm_iint_inode(d_backing_inode(dentry)); + if (iint && (iint->flags & EVM_NEW_FILE)) return 0; /* exception for pseudo filesystems */ @@ -581,6 +572,7 @@ out: * @xattr_name: pointer to the affected extended attribute name * @xattr_value: pointer to the new extended attribute value * @xattr_value_len: pointer to the new extended attribute value length + * @flags: flags to pass into filesystem operations * * Before allowing the 'security.evm' protected xattr to be updated, * verify the existing value is valid. As only the kernel should have @@ -588,9 +580,9 @@ out: * userspace from writing HMAC value. Writing 'security.evm' requires * requires CAP_SYS_ADMIN privileges. */ -int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, - const char *xattr_name, const void *xattr_value, - size_t xattr_value_len) +static int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name, const void *xattr_value, + size_t xattr_value_len, int flags) { const struct evm_ima_xattr_data *xattr_data = xattr_value; @@ -620,8 +612,8 @@ int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, * Removing 'security.evm' requires CAP_SYS_ADMIN privileges and that * the current value is valid. */ -int evm_inode_removexattr(struct mnt_idmap *idmap, - struct dentry *dentry, const char *xattr_name) +static int evm_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name) { /* Policy permits modification of the protected xattrs even though * there's no HMAC key loaded @@ -671,9 +663,11 @@ static inline int evm_inode_set_acl_change(struct mnt_idmap *idmap, * Prevent modifying posix acls causing the EVM HMAC to be re-calculated * and 'security.evm' xattr updated, unless the existing 'security.evm' is * valid. + * + * Return: zero on success, -EPERM on failure. */ -int evm_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, - const char *acl_name, struct posix_acl *kacl) +static int evm_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + const char *acl_name, struct posix_acl *kacl) { enum integrity_status evm_status; @@ -712,11 +706,29 @@ int evm_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, return -EPERM; } +/** + * evm_inode_remove_acl - Protect the EVM extended attribute from posix acls + * @idmap: idmap of the mount + * @dentry: pointer to the affected dentry + * @acl_name: name of the posix acl + * + * Prevent removing posix acls causing the EVM HMAC to be re-calculated + * and 'security.evm' xattr updated, unless the existing 'security.evm' is + * valid. + * + * Return: zero on success, -EPERM on failure. + */ +static int evm_inode_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry, + const char *acl_name) +{ + return evm_inode_set_acl(idmap, dentry, acl_name, NULL); +} + static void evm_reset_status(struct inode *inode) { - struct integrity_iint_cache *iint; + struct evm_iint_cache *iint; - iint = integrity_iint_find(inode); + iint = evm_iint_inode(inode); if (iint) iint->evm_status = INTEGRITY_UNKNOWN; } @@ -752,6 +764,7 @@ bool evm_revalidate_status(const char *xattr_name) * @xattr_name: pointer to the affected extended attribute name * @xattr_value: pointer to the new extended attribute value * @xattr_value_len: pointer to the new extended attribute value length + * @flags: flags to pass into filesystem operations * * Update the HMAC stored in 'security.evm' to reflect the change. * @@ -759,8 +772,11 @@ bool evm_revalidate_status(const char *xattr_name) * __vfs_setxattr_noperm(). The caller of which has taken the inode's * i_mutex lock. */ -void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, - const void *xattr_value, size_t xattr_value_len) +static void evm_inode_post_setxattr(struct dentry *dentry, + const char *xattr_name, + const void *xattr_value, + size_t xattr_value_len, + int flags) { if (!evm_revalidate_status(xattr_name)) return; @@ -779,6 +795,21 @@ void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, evm_update_evmxattr(dentry, xattr_name, xattr_value, xattr_value_len); } +/** + * evm_inode_post_set_acl - Update the EVM extended attribute from posix acls + * @dentry: pointer to the affected dentry + * @acl_name: name of the posix acl + * @kacl: pointer to the posix acls + * + * Update the 'security.evm' xattr with the EVM HMAC re-calculated after setting + * posix acls. + */ +static void evm_inode_post_set_acl(struct dentry *dentry, const char *acl_name, + struct posix_acl *kacl) +{ + return evm_inode_post_setxattr(dentry, acl_name, NULL, 0, 0); +} + /** * evm_inode_post_removexattr - update 'security.evm' after removing the xattr * @dentry: pointer to the affected dentry @@ -789,7 +820,8 @@ void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, * No need to take the i_mutex lock here, as this function is called from * vfs_removexattr() which takes the i_mutex. */ -void evm_inode_post_removexattr(struct dentry *dentry, const char *xattr_name) +static void evm_inode_post_removexattr(struct dentry *dentry, + const char *xattr_name) { if (!evm_revalidate_status(xattr_name)) return; @@ -805,6 +837,22 @@ void evm_inode_post_removexattr(struct dentry *dentry, const char *xattr_name) evm_update_evmxattr(dentry, xattr_name, NULL, 0); } +/** + * evm_inode_post_remove_acl - Update the EVM extended attribute from posix acls + * @idmap: idmap of the mount + * @dentry: pointer to the affected dentry + * @acl_name: name of the posix acl + * + * Update the 'security.evm' xattr with the EVM HMAC re-calculated after + * removing posix acls. + */ +static inline void evm_inode_post_remove_acl(struct mnt_idmap *idmap, + struct dentry *dentry, + const char *acl_name) +{ + evm_inode_post_removexattr(dentry, acl_name); +} + static int evm_attr_change(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr) { @@ -828,8 +876,8 @@ static int evm_attr_change(struct mnt_idmap *idmap, * Permit update of file attributes when files have a valid EVM signature, * except in the case of them having an immutable portable signature. */ -int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, - struct iattr *attr) +static int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) { unsigned int ia_valid = attr->ia_valid; enum integrity_status evm_status; @@ -870,6 +918,7 @@ int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, /** * evm_inode_post_setattr - update 'security.evm' after modifying metadata + * @idmap: idmap of the idmapped mount * @dentry: pointer to the affected dentry * @ia_valid: for the UID and GID status * @@ -879,7 +928,8 @@ int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, * This function is called from notify_change(), which expects the caller * to lock the inode's i_mutex. */ -void evm_inode_post_setattr(struct dentry *dentry, int ia_valid) +static void evm_inode_post_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, int ia_valid) { if (!evm_revalidate_status(NULL)) return; @@ -896,7 +946,7 @@ void evm_inode_post_setattr(struct dentry *dentry, int ia_valid) evm_update_evmxattr(dentry, NULL, NULL, 0); } -int evm_inode_copy_up_xattr(const char *name) +static int evm_inode_copy_up_xattr(const char *name) { if (strcmp(name, XATTR_NAME_EVM) == 0) return 1; /* Discard */ @@ -960,6 +1010,42 @@ out: } EXPORT_SYMBOL_GPL(evm_inode_init_security); +static int evm_inode_alloc_security(struct inode *inode) +{ + struct evm_iint_cache *iint = evm_iint_inode(inode); + + /* Called by security_inode_alloc(), it cannot be NULL. */ + iint->flags = 0UL; + iint->evm_status = INTEGRITY_UNKNOWN; + + return 0; +} + +static void evm_file_release(struct file *file) +{ + struct inode *inode = file_inode(file); + struct evm_iint_cache *iint = evm_iint_inode(inode); + fmode_t mode = file->f_mode; + + if (!S_ISREG(inode->i_mode) || !(mode & FMODE_WRITE)) + return; + + if (iint && atomic_read(&inode->i_writecount) == 1) + iint->flags &= ~EVM_NEW_FILE; +} + +static void evm_post_path_mknod(struct mnt_idmap *idmap, struct dentry *dentry) +{ + struct inode *inode = d_backing_inode(dentry); + struct evm_iint_cache *iint = evm_iint_inode(inode); + + if (!S_ISREG(inode->i_mode)) + return; + + if (iint) + iint->flags |= EVM_NEW_FILE; +} + #ifdef CONFIG_EVM_LOAD_X509 void __init evm_load_x509(void) { @@ -999,4 +1085,45 @@ error: return error; } +static struct security_hook_list evm_hooks[] __ro_after_init = { + LSM_HOOK_INIT(inode_setattr, evm_inode_setattr), + LSM_HOOK_INIT(inode_post_setattr, evm_inode_post_setattr), + LSM_HOOK_INIT(inode_copy_up_xattr, evm_inode_copy_up_xattr), + LSM_HOOK_INIT(inode_setxattr, evm_inode_setxattr), + LSM_HOOK_INIT(inode_post_setxattr, evm_inode_post_setxattr), + LSM_HOOK_INIT(inode_set_acl, evm_inode_set_acl), + LSM_HOOK_INIT(inode_post_set_acl, evm_inode_post_set_acl), + LSM_HOOK_INIT(inode_remove_acl, evm_inode_remove_acl), + LSM_HOOK_INIT(inode_post_remove_acl, evm_inode_post_remove_acl), + LSM_HOOK_INIT(inode_removexattr, evm_inode_removexattr), + LSM_HOOK_INIT(inode_post_removexattr, evm_inode_post_removexattr), + LSM_HOOK_INIT(inode_init_security, evm_inode_init_security), + LSM_HOOK_INIT(inode_alloc_security, evm_inode_alloc_security), + LSM_HOOK_INIT(file_release, evm_file_release), + LSM_HOOK_INIT(path_post_mknod, evm_post_path_mknod), +}; + +static const struct lsm_id evm_lsmid = { + .name = "evm", + .id = LSM_ID_EVM, +}; + +static int __init init_evm_lsm(void) +{ + security_add_hooks(evm_hooks, ARRAY_SIZE(evm_hooks), &evm_lsmid); + return 0; +} + +struct lsm_blob_sizes evm_blob_sizes __ro_after_init = { + .lbs_inode = sizeof(struct evm_iint_cache), + .lbs_xattr_count = 1, +}; + +DEFINE_LSM(evm) = { + .name = "evm", + .init = init_evm_lsm, + .order = LSM_ORDER_LAST, + .blobs = &evm_blob_sizes, +}; + late_initcall(init_evm); diff --git a/security/integrity/iint.c b/security/integrity/iint.c index d4419a2a1e24..068ac6c2ae1e 100644 --- a/security/integrity/iint.c +++ b/security/integrity/iint.c @@ -6,207 +6,14 @@ * Mimi Zohar * * File: integrity_iint.c - * - implements the integrity hooks: integrity_inode_alloc, - * integrity_inode_free - * - cache integrity information associated with an inode - * using a rbtree tree. + * - initialize the integrity directory in securityfs + * - load IMA and EVM keys */ -#include -#include -#include -#include -#include -#include #include -#include #include "integrity.h" -static struct rb_root integrity_iint_tree = RB_ROOT; -static DEFINE_RWLOCK(integrity_iint_lock); -static struct kmem_cache *iint_cache __ro_after_init; - struct dentry *integrity_dir; -/* - * __integrity_iint_find - return the iint associated with an inode - */ -static struct integrity_iint_cache *__integrity_iint_find(struct inode *inode) -{ - struct integrity_iint_cache *iint; - struct rb_node *n = integrity_iint_tree.rb_node; - - while (n) { - iint = rb_entry(n, struct integrity_iint_cache, rb_node); - - if (inode < iint->inode) - n = n->rb_left; - else if (inode > iint->inode) - n = n->rb_right; - else - return iint; - } - - return NULL; -} - -/* - * integrity_iint_find - return the iint associated with an inode - */ -struct integrity_iint_cache *integrity_iint_find(struct inode *inode) -{ - struct integrity_iint_cache *iint; - - if (!IS_IMA(inode)) - return NULL; - - read_lock(&integrity_iint_lock); - iint = __integrity_iint_find(inode); - read_unlock(&integrity_iint_lock); - - return iint; -} - -#define IMA_MAX_NESTING (FILESYSTEM_MAX_STACK_DEPTH+1) - -/* - * It is not clear that IMA should be nested at all, but as long is it measures - * files both on overlayfs and on underlying fs, we need to annotate the iint - * mutex to avoid lockdep false positives related to IMA + overlayfs. - * See ovl_lockdep_annotate_inode_mutex_key() for more details. - */ -static inline void iint_lockdep_annotate(struct integrity_iint_cache *iint, - struct inode *inode) -{ -#ifdef CONFIG_LOCKDEP - static struct lock_class_key iint_mutex_key[IMA_MAX_NESTING]; - - int depth = inode->i_sb->s_stack_depth; - - if (WARN_ON_ONCE(depth < 0 || depth >= IMA_MAX_NESTING)) - depth = 0; - - lockdep_set_class(&iint->mutex, &iint_mutex_key[depth]); -#endif -} - -static void iint_init_always(struct integrity_iint_cache *iint, - struct inode *inode) -{ - iint->ima_hash = NULL; - iint->version = 0; - iint->flags = 0UL; - iint->atomic_flags = 0UL; - iint->ima_file_status = INTEGRITY_UNKNOWN; - iint->ima_mmap_status = INTEGRITY_UNKNOWN; - iint->ima_bprm_status = INTEGRITY_UNKNOWN; - iint->ima_read_status = INTEGRITY_UNKNOWN; - iint->ima_creds_status = INTEGRITY_UNKNOWN; - iint->evm_status = INTEGRITY_UNKNOWN; - iint->measured_pcrs = 0; - mutex_init(&iint->mutex); - iint_lockdep_annotate(iint, inode); -} - -static void iint_free(struct integrity_iint_cache *iint) -{ - kfree(iint->ima_hash); - mutex_destroy(&iint->mutex); - kmem_cache_free(iint_cache, iint); -} - -/** - * integrity_inode_get - find or allocate an iint associated with an inode - * @inode: pointer to the inode - * @return: allocated iint - * - * Caller must lock i_mutex - */ -struct integrity_iint_cache *integrity_inode_get(struct inode *inode) -{ - struct rb_node **p; - struct rb_node *node, *parent = NULL; - struct integrity_iint_cache *iint, *test_iint; - - iint = integrity_iint_find(inode); - if (iint) - return iint; - - iint = kmem_cache_alloc(iint_cache, GFP_NOFS); - if (!iint) - return NULL; - - iint_init_always(iint, inode); - - write_lock(&integrity_iint_lock); - - p = &integrity_iint_tree.rb_node; - while (*p) { - parent = *p; - test_iint = rb_entry(parent, struct integrity_iint_cache, - rb_node); - if (inode < test_iint->inode) { - p = &(*p)->rb_left; - } else if (inode > test_iint->inode) { - p = &(*p)->rb_right; - } else { - write_unlock(&integrity_iint_lock); - kmem_cache_free(iint_cache, iint); - return test_iint; - } - } - - iint->inode = inode; - node = &iint->rb_node; - inode->i_flags |= S_IMA; - rb_link_node(node, parent, p); - rb_insert_color(node, &integrity_iint_tree); - - write_unlock(&integrity_iint_lock); - return iint; -} - -/** - * integrity_inode_free - called on security_inode_free - * @inode: pointer to the inode - * - * Free the integrity information(iint) associated with an inode. - */ -void integrity_inode_free(struct inode *inode) -{ - struct integrity_iint_cache *iint; - - if (!IS_IMA(inode)) - return; - - write_lock(&integrity_iint_lock); - iint = __integrity_iint_find(inode); - rb_erase(&iint->rb_node, &integrity_iint_tree); - write_unlock(&integrity_iint_lock); - - iint_free(iint); -} - -static void iint_init_once(void *foo) -{ - struct integrity_iint_cache *iint = (struct integrity_iint_cache *) foo; - - memset(iint, 0, sizeof(*iint)); -} - -static int __init integrity_iintcache_init(void) -{ - iint_cache = - kmem_cache_create("iint_cache", sizeof(struct integrity_iint_cache), - 0, SLAB_PANIC, iint_init_once); - return 0; -} -DEFINE_LSM(integrity) = { - .name = "integrity", - .init = integrity_iintcache_init, - .order = LSM_ORDER_LAST, -}; - - /* * integrity_kernel_read - read data from the file * diff --git a/security/integrity/ima/Kconfig b/security/integrity/ima/Kconfig index b98bfe9efd0c..475c32615006 100644 --- a/security/integrity/ima/Kconfig +++ b/security/integrity/ima/Kconfig @@ -8,6 +8,7 @@ config IMA select CRYPTO_HMAC select CRYPTO_SHA1 select CRYPTO_HASH_INFO + select SECURITY_PATH select TCG_TPM if HAS_IOMEM select TCG_TIS if TCG_TPM && X86 select TCG_CRB if TCG_TPM && ACPI diff --git a/security/integrity/ima/Makefile b/security/integrity/ima/Makefile index 2499f2485c04..b376d38b4ee6 100644 --- a/security/integrity/ima/Makefile +++ b/security/integrity/ima/Makefile @@ -4,7 +4,7 @@ # Measurement Architecture(IMA). # -obj-$(CONFIG_IMA) += ima.o +obj-$(CONFIG_IMA) += ima.o ima_iint.o ima-y := ima_fs.o ima_queue.o ima_init.o ima_main.o ima_crypto.o ima_api.o \ ima_policy.o ima_template.o ima_template_lib.o diff --git a/security/integrity/ima/ima.h b/security/integrity/ima/ima.h index c29db699c996..11d7c0332207 100644 --- a/security/integrity/ima/ima.h +++ b/security/integrity/ima/ima.h @@ -60,7 +60,7 @@ extern const char boot_aggregate_name[]; /* IMA event related data */ struct ima_event_data { - struct integrity_iint_cache *iint; + struct ima_iint_cache *iint; struct file *file; const unsigned char *filename; struct evm_ima_xattr_data *xattr_value; @@ -119,6 +119,107 @@ struct ima_kexec_hdr { u64 count; }; +/* IMA iint action cache flags */ +#define IMA_MEASURE 0x00000001 +#define IMA_MEASURED 0x00000002 +#define IMA_APPRAISE 0x00000004 +#define IMA_APPRAISED 0x00000008 +/*#define IMA_COLLECT 0x00000010 do not use this flag */ +#define IMA_COLLECTED 0x00000020 +#define IMA_AUDIT 0x00000040 +#define IMA_AUDITED 0x00000080 +#define IMA_HASH 0x00000100 +#define IMA_HASHED 0x00000200 + +/* IMA iint policy rule cache flags */ +#define IMA_NONACTION_FLAGS 0xff000000 +#define IMA_DIGSIG_REQUIRED 0x01000000 +#define IMA_PERMIT_DIRECTIO 0x02000000 +#define IMA_NEW_FILE 0x04000000 +#define IMA_FAIL_UNVERIFIABLE_SIGS 0x10000000 +#define IMA_MODSIG_ALLOWED 0x20000000 +#define IMA_CHECK_BLACKLIST 0x40000000 +#define IMA_VERITY_REQUIRED 0x80000000 + +#define IMA_DO_MASK (IMA_MEASURE | IMA_APPRAISE | IMA_AUDIT | \ + IMA_HASH | IMA_APPRAISE_SUBMASK) +#define IMA_DONE_MASK (IMA_MEASURED | IMA_APPRAISED | IMA_AUDITED | \ + IMA_HASHED | IMA_COLLECTED | \ + IMA_APPRAISED_SUBMASK) + +/* IMA iint subaction appraise cache flags */ +#define IMA_FILE_APPRAISE 0x00001000 +#define IMA_FILE_APPRAISED 0x00002000 +#define IMA_MMAP_APPRAISE 0x00004000 +#define IMA_MMAP_APPRAISED 0x00008000 +#define IMA_BPRM_APPRAISE 0x00010000 +#define IMA_BPRM_APPRAISED 0x00020000 +#define IMA_READ_APPRAISE 0x00040000 +#define IMA_READ_APPRAISED 0x00080000 +#define IMA_CREDS_APPRAISE 0x00100000 +#define IMA_CREDS_APPRAISED 0x00200000 +#define IMA_APPRAISE_SUBMASK (IMA_FILE_APPRAISE | IMA_MMAP_APPRAISE | \ + IMA_BPRM_APPRAISE | IMA_READ_APPRAISE | \ + IMA_CREDS_APPRAISE) +#define IMA_APPRAISED_SUBMASK (IMA_FILE_APPRAISED | IMA_MMAP_APPRAISED | \ + IMA_BPRM_APPRAISED | IMA_READ_APPRAISED | \ + IMA_CREDS_APPRAISED) + +/* IMA iint cache atomic_flags */ +#define IMA_CHANGE_XATTR 0 +#define IMA_UPDATE_XATTR 1 +#define IMA_CHANGE_ATTR 2 +#define IMA_DIGSIG 3 +#define IMA_MUST_MEASURE 4 + +/* IMA integrity metadata associated with an inode */ +struct ima_iint_cache { + struct mutex mutex; /* protects: version, flags, digest */ + u64 version; /* track inode changes */ + unsigned long flags; + unsigned long measured_pcrs; + unsigned long atomic_flags; + unsigned long real_ino; + dev_t real_dev; + enum integrity_status ima_file_status:4; + enum integrity_status ima_mmap_status:4; + enum integrity_status ima_bprm_status:4; + enum integrity_status ima_read_status:4; + enum integrity_status ima_creds_status:4; + struct ima_digest_data *ima_hash; +}; + +extern struct lsm_blob_sizes ima_blob_sizes; + +static inline struct ima_iint_cache * +ima_inode_get_iint(const struct inode *inode) +{ + struct ima_iint_cache **iint_sec; + + if (unlikely(!inode->i_security)) + return NULL; + + iint_sec = inode->i_security + ima_blob_sizes.lbs_inode; + return *iint_sec; +} + +static inline void ima_inode_set_iint(const struct inode *inode, + struct ima_iint_cache *iint) +{ + struct ima_iint_cache **iint_sec; + + if (unlikely(!inode->i_security)) + return; + + iint_sec = inode->i_security + ima_blob_sizes.lbs_inode; + *iint_sec = iint; +} + +struct ima_iint_cache *ima_iint_find(struct inode *inode); +struct ima_iint_cache *ima_inode_get(struct inode *inode); +void ima_inode_free(struct inode *inode); +void __init ima_iintcache_init(void); + extern const int read_idmap[]; #ifdef CONFIG_HAVE_IMA_KEXEC @@ -127,6 +228,12 @@ void ima_load_kexec_buffer(void); static inline void ima_load_kexec_buffer(void) {} #endif /* CONFIG_HAVE_IMA_KEXEC */ +#ifdef CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS +void ima_post_key_create_or_update(struct key *keyring, struct key *key, + const void *payload, size_t plen, + unsigned long flags, bool create); +#endif + /* * The default binary_runtime_measurements list format is defined as the * platform native format. The canonical format is defined as little-endian. @@ -146,8 +253,8 @@ int ima_calc_field_array_hash(struct ima_field_data *field_data, struct ima_template_entry *entry); int ima_calc_boot_aggregate(struct ima_digest_data *hash); void ima_add_violation(struct file *file, const unsigned char *filename, - struct integrity_iint_cache *iint, - const char *op, const char *cause); + struct ima_iint_cache *iint, const char *op, + const char *cause); int ima_init_crypto(void); void ima_putc(struct seq_file *m, void *data, int datalen); void ima_print_digest(struct seq_file *m, u8 *digest, u32 size); @@ -261,10 +368,10 @@ int ima_get_action(struct mnt_idmap *idmap, struct inode *inode, struct ima_template_desc **template_desc, const char *func_data, unsigned int *allowed_algos); int ima_must_measure(struct inode *inode, int mask, enum ima_hooks func); -int ima_collect_measurement(struct integrity_iint_cache *iint, - struct file *file, void *buf, loff_t size, - enum hash_algo algo, struct modsig *modsig); -void ima_store_measurement(struct integrity_iint_cache *iint, struct file *file, +int ima_collect_measurement(struct ima_iint_cache *iint, struct file *file, + void *buf, loff_t size, enum hash_algo algo, + struct modsig *modsig); +void ima_store_measurement(struct ima_iint_cache *iint, struct file *file, const unsigned char *filename, struct evm_ima_xattr_data *xattr_value, int xattr_len, const struct modsig *modsig, int pcr, @@ -274,7 +381,7 @@ int process_buffer_measurement(struct mnt_idmap *idmap, const char *eventname, enum ima_hooks func, int pcr, const char *func_data, bool buf_hash, u8 *digest, size_t digest_len); -void ima_audit_measurement(struct integrity_iint_cache *iint, +void ima_audit_measurement(struct ima_iint_cache *iint, const unsigned char *filename); int ima_alloc_init_template(struct ima_event_data *event_data, struct ima_template_entry **entry, @@ -312,32 +419,32 @@ int ima_policy_show(struct seq_file *m, void *v); #define IMA_APPRAISE_KEXEC 0x40 #ifdef CONFIG_IMA_APPRAISE -int ima_check_blacklist(struct integrity_iint_cache *iint, +int ima_check_blacklist(struct ima_iint_cache *iint, const struct modsig *modsig, int pcr); -int ima_appraise_measurement(enum ima_hooks func, - struct integrity_iint_cache *iint, +int ima_appraise_measurement(enum ima_hooks func, struct ima_iint_cache *iint, struct file *file, const unsigned char *filename, struct evm_ima_xattr_data *xattr_value, int xattr_len, const struct modsig *modsig); int ima_must_appraise(struct mnt_idmap *idmap, struct inode *inode, int mask, enum ima_hooks func); -void ima_update_xattr(struct integrity_iint_cache *iint, struct file *file); -enum integrity_status ima_get_cache_status(struct integrity_iint_cache *iint, +void ima_update_xattr(struct ima_iint_cache *iint, struct file *file); +enum integrity_status ima_get_cache_status(struct ima_iint_cache *iint, enum ima_hooks func); enum hash_algo ima_get_hash_algo(const struct evm_ima_xattr_data *xattr_value, int xattr_len); int ima_read_xattr(struct dentry *dentry, struct evm_ima_xattr_data **xattr_value, int xattr_len); +void __init init_ima_appraise_lsm(const struct lsm_id *lsmid); #else -static inline int ima_check_blacklist(struct integrity_iint_cache *iint, +static inline int ima_check_blacklist(struct ima_iint_cache *iint, const struct modsig *modsig, int pcr) { return 0; } static inline int ima_appraise_measurement(enum ima_hooks func, - struct integrity_iint_cache *iint, + struct ima_iint_cache *iint, struct file *file, const unsigned char *filename, struct evm_ima_xattr_data *xattr_value, @@ -354,14 +461,13 @@ static inline int ima_must_appraise(struct mnt_idmap *idmap, return 0; } -static inline void ima_update_xattr(struct integrity_iint_cache *iint, +static inline void ima_update_xattr(struct ima_iint_cache *iint, struct file *file) { } -static inline enum integrity_status ima_get_cache_status(struct integrity_iint_cache - *iint, - enum ima_hooks func) +static inline enum integrity_status +ima_get_cache_status(struct ima_iint_cache *iint, enum ima_hooks func) { return INTEGRITY_UNKNOWN; } @@ -379,6 +485,10 @@ static inline int ima_read_xattr(struct dentry *dentry, return 0; } +static inline void __init init_ima_appraise_lsm(const struct lsm_id *lsmid) +{ +} + #endif /* CONFIG_IMA_APPRAISE */ #ifdef CONFIG_IMA_APPRAISE_MODSIG diff --git a/security/integrity/ima/ima_api.c b/security/integrity/ima/ima_api.c index 597ea0c4d72f..b37d043d5748 100644 --- a/security/integrity/ima/ima_api.c +++ b/security/integrity/ima/ima_api.c @@ -131,8 +131,8 @@ int ima_store_template(struct ima_template_entry *entry, * value is invalidated. */ void ima_add_violation(struct file *file, const unsigned char *filename, - struct integrity_iint_cache *iint, - const char *op, const char *cause) + struct ima_iint_cache *iint, const char *op, + const char *cause) { struct ima_template_entry *entry; struct inode *inode = file_inode(file); @@ -201,7 +201,8 @@ int ima_get_action(struct mnt_idmap *idmap, struct inode *inode, allowed_algos); } -static bool ima_get_verity_digest(struct integrity_iint_cache *iint, +static bool ima_get_verity_digest(struct ima_iint_cache *iint, + struct inode *inode, struct ima_max_digest_data *hash) { enum hash_algo alg; @@ -211,7 +212,7 @@ static bool ima_get_verity_digest(struct integrity_iint_cache *iint, * On failure, 'measure' policy rules will result in a file data * hash containing 0's. */ - digest_len = fsverity_get_digest(iint->inode, hash->digest, NULL, &alg); + digest_len = fsverity_get_digest(inode, hash->digest, NULL, &alg); if (digest_len == 0) return false; @@ -237,9 +238,9 @@ static bool ima_get_verity_digest(struct integrity_iint_cache *iint, * * Return 0 on success, error code otherwise */ -int ima_collect_measurement(struct integrity_iint_cache *iint, - struct file *file, void *buf, loff_t size, - enum hash_algo algo, struct modsig *modsig) +int ima_collect_measurement(struct ima_iint_cache *iint, struct file *file, + void *buf, loff_t size, enum hash_algo algo, + struct modsig *modsig) { const char *audit_cause = "failed"; struct inode *inode = file_inode(file); @@ -280,7 +281,7 @@ int ima_collect_measurement(struct integrity_iint_cache *iint, memset(&hash.digest, 0, sizeof(hash.digest)); if (iint->flags & IMA_VERITY_REQUIRED) { - if (!ima_get_verity_digest(iint, &hash)) { + if (!ima_get_verity_digest(iint, inode, &hash)) { audit_cause = "no-verity-digest"; result = -ENODATA; } @@ -338,8 +339,8 @@ out: * * Must be called with iint->mutex held. */ -void ima_store_measurement(struct integrity_iint_cache *iint, - struct file *file, const unsigned char *filename, +void ima_store_measurement(struct ima_iint_cache *iint, struct file *file, + const unsigned char *filename, struct evm_ima_xattr_data *xattr_value, int xattr_len, const struct modsig *modsig, int pcr, struct ima_template_desc *template_desc) @@ -382,7 +383,7 @@ void ima_store_measurement(struct integrity_iint_cache *iint, ima_free_template_entry(entry); } -void ima_audit_measurement(struct integrity_iint_cache *iint, +void ima_audit_measurement(struct ima_iint_cache *iint, const unsigned char *filename) { struct audit_buffer *ab; diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index 870dde67707b..3497741caea9 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -84,8 +84,7 @@ int ima_must_appraise(struct mnt_idmap *idmap, struct inode *inode, NULL, NULL, NULL); } -static int ima_fix_xattr(struct dentry *dentry, - struct integrity_iint_cache *iint) +static int ima_fix_xattr(struct dentry *dentry, struct ima_iint_cache *iint) { int rc, offset; u8 algo = iint->ima_hash->algo; @@ -106,7 +105,7 @@ static int ima_fix_xattr(struct dentry *dentry, } /* Return specific func appraised cached result */ -enum integrity_status ima_get_cache_status(struct integrity_iint_cache *iint, +enum integrity_status ima_get_cache_status(struct ima_iint_cache *iint, enum ima_hooks func) { switch (func) { @@ -126,7 +125,7 @@ enum integrity_status ima_get_cache_status(struct integrity_iint_cache *iint, } } -static void ima_set_cache_status(struct integrity_iint_cache *iint, +static void ima_set_cache_status(struct ima_iint_cache *iint, enum ima_hooks func, enum integrity_status status) { @@ -152,8 +151,7 @@ static void ima_set_cache_status(struct integrity_iint_cache *iint, } } -static void ima_cache_flags(struct integrity_iint_cache *iint, - enum ima_hooks func) +static void ima_cache_flags(struct ima_iint_cache *iint, enum ima_hooks func) { switch (func) { case MMAP_CHECK: @@ -276,7 +274,7 @@ static int calc_file_id_hash(enum evm_ima_xattr_type type, * * Return 0 on success, error code otherwise. */ -static int xattr_verify(enum ima_hooks func, struct integrity_iint_cache *iint, +static int xattr_verify(enum ima_hooks func, struct ima_iint_cache *iint, struct evm_ima_xattr_data *xattr_value, int xattr_len, enum integrity_status *status, const char **cause) { @@ -443,7 +441,7 @@ static int modsig_verify(enum ima_hooks func, const struct modsig *modsig, * * Returns -EPERM if the hash is blacklisted. */ -int ima_check_blacklist(struct integrity_iint_cache *iint, +int ima_check_blacklist(struct ima_iint_cache *iint, const struct modsig *modsig, int pcr) { enum hash_algo hash_algo; @@ -477,8 +475,7 @@ int ima_check_blacklist(struct integrity_iint_cache *iint, * * Return 0 on success, error code otherwise */ -int ima_appraise_measurement(enum ima_hooks func, - struct integrity_iint_cache *iint, +int ima_appraise_measurement(enum ima_hooks func, struct ima_iint_cache *iint, struct file *file, const unsigned char *filename, struct evm_ima_xattr_data *xattr_value, int xattr_len, const struct modsig *modsig) @@ -520,7 +517,7 @@ int ima_appraise_measurement(enum ima_hooks func, } status = evm_verifyxattr(dentry, XATTR_NAME_IMA, xattr_value, - rc < 0 ? 0 : rc, iint); + rc < 0 ? 0 : rc); switch (status) { case INTEGRITY_PASS: case INTEGRITY_PASS_IMMUTABLE: @@ -603,7 +600,7 @@ out: /* * ima_update_xattr - update 'security.ima' hash value */ -void ima_update_xattr(struct integrity_iint_cache *iint, struct file *file) +void ima_update_xattr(struct ima_iint_cache *iint, struct file *file) { struct dentry *dentry = file_dentry(file); int rc = 0; @@ -629,17 +626,18 @@ void ima_update_xattr(struct integrity_iint_cache *iint, struct file *file) * ima_inode_post_setattr - reflect file metadata changes * @idmap: idmap of the mount the inode was found from * @dentry: pointer to the affected dentry + * @ia_valid: for the UID and GID status * * Changes to a dentry's metadata might result in needing to appraise. * * This function is called from notify_change(), which expects the caller * to lock the inode's i_mutex. */ -void ima_inode_post_setattr(struct mnt_idmap *idmap, - struct dentry *dentry) +static void ima_inode_post_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, int ia_valid) { struct inode *inode = d_backing_inode(dentry); - struct integrity_iint_cache *iint; + struct ima_iint_cache *iint; int action; if (!(ima_policy_flag & IMA_APPRAISE) || !S_ISREG(inode->i_mode) @@ -647,7 +645,7 @@ void ima_inode_post_setattr(struct mnt_idmap *idmap, return; action = ima_must_appraise(idmap, inode, MAY_ACCESS, POST_SETATTR); - iint = integrity_iint_find(inode); + iint = ima_iint_find(inode); if (iint) { set_bit(IMA_CHANGE_ATTR, &iint->atomic_flags); if (!action) @@ -673,12 +671,12 @@ static int ima_protect_xattr(struct dentry *dentry, const char *xattr_name, static void ima_reset_appraise_flags(struct inode *inode, int digsig) { - struct integrity_iint_cache *iint; + struct ima_iint_cache *iint; if (!(ima_policy_flag & IMA_APPRAISE) || !S_ISREG(inode->i_mode)) return; - iint = integrity_iint_find(inode); + iint = ima_iint_find(inode); if (!iint) return; iint->measured_pcrs = 0; @@ -749,8 +747,9 @@ static int validate_hash_algo(struct dentry *dentry, return -EACCES; } -int ima_inode_setxattr(struct dentry *dentry, const char *xattr_name, - const void *xattr_value, size_t xattr_value_len) +static int ima_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name, const void *xattr_value, + size_t xattr_value_len, int flags) { const struct evm_ima_xattr_data *xvalue = xattr_value; int digsig = 0; @@ -779,8 +778,8 @@ int ima_inode_setxattr(struct dentry *dentry, const char *xattr_name, return result; } -int ima_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, - const char *acl_name, struct posix_acl *kacl) +static int ima_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + const char *acl_name, struct posix_acl *kacl) { if (evm_revalidate_status(acl_name)) ima_reset_appraise_flags(d_backing_inode(dentry), 0); @@ -788,7 +787,8 @@ int ima_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, return 0; } -int ima_inode_removexattr(struct dentry *dentry, const char *xattr_name) +static int ima_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *xattr_name) { int result; @@ -800,3 +800,23 @@ int ima_inode_removexattr(struct dentry *dentry, const char *xattr_name) } return result; } + +static int ima_inode_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry, + const char *acl_name) +{ + return ima_inode_set_acl(idmap, dentry, acl_name, NULL); +} + +static struct security_hook_list ima_appraise_hooks[] __ro_after_init = { + LSM_HOOK_INIT(inode_post_setattr, ima_inode_post_setattr), + LSM_HOOK_INIT(inode_setxattr, ima_inode_setxattr), + LSM_HOOK_INIT(inode_set_acl, ima_inode_set_acl), + LSM_HOOK_INIT(inode_removexattr, ima_inode_removexattr), + LSM_HOOK_INIT(inode_remove_acl, ima_inode_remove_acl), +}; + +void __init init_ima_appraise_lsm(const struct lsm_id *lsmid) +{ + security_add_hooks(ima_appraise_hooks, ARRAY_SIZE(ima_appraise_hooks), + lsmid); +} diff --git a/security/integrity/ima/ima_iint.c b/security/integrity/ima/ima_iint.c new file mode 100644 index 000000000000..e7c9c216c1c6 --- /dev/null +++ b/security/integrity/ima/ima_iint.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2008 IBM Corporation + * + * Authors: + * Mimi Zohar + * + * File: ima_iint.c + * - implements the IMA hook: ima_inode_free + * - cache integrity information in the inode security blob + */ +#include + +#include "ima.h" + +static struct kmem_cache *ima_iint_cache __ro_after_init; + +/** + * ima_iint_find - Return the iint associated with an inode + * @inode: Pointer to the inode + * + * Return the IMA integrity information (iint) associated with an inode, if the + * inode was processed by IMA. + * + * Return: Found iint or NULL. + */ +struct ima_iint_cache *ima_iint_find(struct inode *inode) +{ + if (!IS_IMA(inode)) + return NULL; + + return ima_inode_get_iint(inode); +} + +#define IMA_MAX_NESTING (FILESYSTEM_MAX_STACK_DEPTH + 1) + +/* + * It is not clear that IMA should be nested at all, but as long is it measures + * files both on overlayfs and on underlying fs, we need to annotate the iint + * mutex to avoid lockdep false positives related to IMA + overlayfs. + * See ovl_lockdep_annotate_inode_mutex_key() for more details. + */ +static inline void ima_iint_lockdep_annotate(struct ima_iint_cache *iint, + struct inode *inode) +{ +#ifdef CONFIG_LOCKDEP + static struct lock_class_key ima_iint_mutex_key[IMA_MAX_NESTING]; + + int depth = inode->i_sb->s_stack_depth; + + if (WARN_ON_ONCE(depth < 0 || depth >= IMA_MAX_NESTING)) + depth = 0; + + lockdep_set_class(&iint->mutex, &ima_iint_mutex_key[depth]); +#endif +} + +static void ima_iint_init_always(struct ima_iint_cache *iint, + struct inode *inode) +{ + iint->ima_hash = NULL; + iint->version = 0; + iint->flags = 0UL; + iint->atomic_flags = 0UL; + iint->ima_file_status = INTEGRITY_UNKNOWN; + iint->ima_mmap_status = INTEGRITY_UNKNOWN; + iint->ima_bprm_status = INTEGRITY_UNKNOWN; + iint->ima_read_status = INTEGRITY_UNKNOWN; + iint->ima_creds_status = INTEGRITY_UNKNOWN; + iint->measured_pcrs = 0; + mutex_init(&iint->mutex); + ima_iint_lockdep_annotate(iint, inode); +} + +static void ima_iint_free(struct ima_iint_cache *iint) +{ + kfree(iint->ima_hash); + mutex_destroy(&iint->mutex); + kmem_cache_free(ima_iint_cache, iint); +} + +/** + * ima_inode_get - Find or allocate an iint associated with an inode + * @inode: Pointer to the inode + * + * Find an iint associated with an inode, and allocate a new one if not found. + * Caller must lock i_mutex. + * + * Return: An iint on success, NULL on error. + */ +struct ima_iint_cache *ima_inode_get(struct inode *inode) +{ + struct ima_iint_cache *iint; + + iint = ima_iint_find(inode); + if (iint) + return iint; + + iint = kmem_cache_alloc(ima_iint_cache, GFP_NOFS); + if (!iint) + return NULL; + + ima_iint_init_always(iint, inode); + + inode->i_flags |= S_IMA; + ima_inode_set_iint(inode, iint); + + return iint; +} + +/** + * ima_inode_free - Called on inode free + * @inode: Pointer to the inode + * + * Free the iint associated with an inode. + */ +void ima_inode_free(struct inode *inode) +{ + struct ima_iint_cache *iint; + + if (!IS_IMA(inode)) + return; + + iint = ima_iint_find(inode); + ima_inode_set_iint(inode, NULL); + + ima_iint_free(iint); +} + +static void ima_iint_init_once(void *foo) +{ + struct ima_iint_cache *iint = (struct ima_iint_cache *)foo; + + memset(iint, 0, sizeof(*iint)); +} + +void __init ima_iintcache_init(void) +{ + ima_iint_cache = + kmem_cache_create("ima_iint_cache", sizeof(struct ima_iint_cache), + 0, SLAB_PANIC, ima_iint_init_once); +} diff --git a/security/integrity/ima/ima_init.c b/security/integrity/ima/ima_init.c index 63979aefc95f..393f5c7912d5 100644 --- a/security/integrity/ima/ima_init.c +++ b/security/integrity/ima/ima_init.c @@ -44,7 +44,7 @@ static int __init ima_add_boot_aggregate(void) static const char op[] = "add_boot_aggregate"; const char *audit_cause = "ENOMEM"; struct ima_template_entry *entry; - struct integrity_iint_cache tmp_iint, *iint = &tmp_iint; + struct ima_iint_cache tmp_iint, *iint = &tmp_iint; struct ima_event_data event_data = { .iint = iint, .filename = boot_aggregate_name }; struct ima_max_digest_data hash; diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index cc1217ac2c6f..c84e8c55333d 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -114,7 +114,7 @@ static int mmap_violation_check(enum ima_hooks func, struct file *file, * */ static void ima_rdwr_violation_check(struct file *file, - struct integrity_iint_cache *iint, + struct ima_iint_cache *iint, int must_measure, char **pathbuf, const char **pathname, @@ -127,7 +127,7 @@ static void ima_rdwr_violation_check(struct file *file, if (mode & FMODE_WRITE) { if (atomic_read(&inode->i_readcount) && IS_IMA(inode)) { if (!iint) - iint = integrity_iint_find(inode); + iint = ima_iint_find(inode); /* IMA_MEASURE is set from reader side */ if (iint && test_bit(IMA_MUST_MEASURE, &iint->atomic_flags)) @@ -153,7 +153,7 @@ static void ima_rdwr_violation_check(struct file *file, "invalid_pcr", "open_writers"); } -static void ima_check_last_writer(struct integrity_iint_cache *iint, +static void ima_check_last_writer(struct ima_iint_cache *iint, struct inode *inode, struct file *file) { fmode_t mode = file->f_mode; @@ -189,15 +189,15 @@ static void ima_check_last_writer(struct integrity_iint_cache *iint, * * Flag files that changed, based on i_version */ -void ima_file_free(struct file *file) +static void ima_file_free(struct file *file) { struct inode *inode = file_inode(file); - struct integrity_iint_cache *iint; + struct ima_iint_cache *iint; if (!ima_policy_flag || !S_ISREG(inode->i_mode)) return; - iint = integrity_iint_find(inode); + iint = ima_iint_find(inode); if (!iint) return; @@ -209,7 +209,7 @@ static int process_measurement(struct file *file, const struct cred *cred, enum ima_hooks func) { struct inode *backing_inode, *inode = file_inode(file); - struct integrity_iint_cache *iint = NULL; + struct ima_iint_cache *iint = NULL; struct ima_template_desc *template_desc = NULL; char *pathbuf = NULL; char filename[NAME_MAX]; @@ -248,7 +248,7 @@ static int process_measurement(struct file *file, const struct cred *cred, inode_lock(inode); if (action) { - iint = integrity_inode_get(inode); + iint = ima_inode_get(inode); if (!iint) rc = -ENOMEM; } @@ -427,8 +427,8 @@ out: * On success return 0. On integrity appraisal error, assuming the file * is in policy and IMA-appraisal is in enforcing mode, return -EACCES. */ -int ima_file_mmap(struct file *file, unsigned long reqprot, - unsigned long prot, unsigned long flags) +static int ima_file_mmap(struct file *file, unsigned long reqprot, + unsigned long prot, unsigned long flags) { u32 secid; int ret; @@ -455,7 +455,8 @@ int ima_file_mmap(struct file *file, unsigned long reqprot, /** * ima_file_mprotect - based on policy, limit mprotect change * @vma: vm_area_struct protection is set to - * @prot: contains the protection that will be applied by the kernel. + * @reqprot: protection requested by the application + * @prot: protection that will be applied by the kernel * * Files can be mmap'ed read/write and later changed to execute to circumvent * IMA's mmap appraisal policy rules. Due to locking issues (mmap semaphore @@ -465,7 +466,8 @@ int ima_file_mmap(struct file *file, unsigned long reqprot, * * On mprotect change success, return 0. On failure, return -EACESS. */ -int ima_file_mprotect(struct vm_area_struct *vma, unsigned long prot) +static int ima_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, + unsigned long prot) { struct ima_template_desc *template = NULL; struct file *file; @@ -523,7 +525,7 @@ int ima_file_mprotect(struct vm_area_struct *vma, unsigned long prot) * On success return 0. On integrity appraisal error, assuming the file * is in policy and IMA-appraisal is in enforcing mode, return -EACCES. */ -int ima_bprm_check(struct linux_binprm *bprm) +static int ima_bprm_check(struct linux_binprm *bprm) { int ret; u32 secid; @@ -549,7 +551,7 @@ int ima_bprm_check(struct linux_binprm *bprm) * On success return 0. On integrity appraisal error, assuming the file * is in policy and IMA-appraisal is in enforcing mode, return -EACCES. */ -int ima_file_check(struct file *file, int mask) +static int ima_file_check(struct file *file, int mask) { u32 secid; @@ -558,16 +560,15 @@ int ima_file_check(struct file *file, int mask) mask & (MAY_READ | MAY_WRITE | MAY_EXEC | MAY_APPEND), FILE_CHECK); } -EXPORT_SYMBOL_GPL(ima_file_check); static int __ima_inode_hash(struct inode *inode, struct file *file, char *buf, size_t buf_size) { - struct integrity_iint_cache *iint = NULL, tmp_iint; + struct ima_iint_cache *iint = NULL, tmp_iint; int rc, hash_algo; if (ima_policy_flag) { - iint = integrity_iint_find(inode); + iint = ima_iint_find(inode); if (iint) mutex_lock(&iint->mutex); } @@ -577,7 +578,6 @@ static int __ima_inode_hash(struct inode *inode, struct file *file, char *buf, mutex_unlock(&iint->mutex); memset(&tmp_iint, 0, sizeof(tmp_iint)); - tmp_iint.inode = inode; mutex_init(&tmp_iint.mutex); rc = ima_collect_measurement(&tmp_iint, file, NULL, 0, @@ -683,10 +683,11 @@ EXPORT_SYMBOL_GPL(ima_inode_hash); * Skip calling process_measurement(), but indicate which newly, created * tmpfiles are in policy. */ -void ima_post_create_tmpfile(struct mnt_idmap *idmap, - struct inode *inode) +static void ima_post_create_tmpfile(struct mnt_idmap *idmap, + struct inode *inode) + { - struct integrity_iint_cache *iint; + struct ima_iint_cache *iint; int must_appraise; if (!ima_policy_flag || !S_ISREG(inode->i_mode)) @@ -698,7 +699,7 @@ void ima_post_create_tmpfile(struct mnt_idmap *idmap, return; /* Nothing to do if we can't allocate memory */ - iint = integrity_inode_get(inode); + iint = ima_inode_get(inode); if (!iint) return; @@ -715,10 +716,9 @@ void ima_post_create_tmpfile(struct mnt_idmap *idmap, * Mark files created via the mknodat syscall as new, so that the * file data can be written later. */ -void ima_post_path_mknod(struct mnt_idmap *idmap, - struct dentry *dentry) +static void ima_post_path_mknod(struct mnt_idmap *idmap, struct dentry *dentry) { - struct integrity_iint_cache *iint; + struct ima_iint_cache *iint; struct inode *inode = dentry->d_inode; int must_appraise; @@ -731,7 +731,7 @@ void ima_post_path_mknod(struct mnt_idmap *idmap, return; /* Nothing to do if we can't allocate memory */ - iint = integrity_inode_get(inode); + iint = ima_inode_get(inode); if (!iint) return; @@ -751,8 +751,8 @@ void ima_post_path_mknod(struct mnt_idmap *idmap, * * For permission return 0, otherwise return -EACCES. */ -int ima_read_file(struct file *file, enum kernel_read_file_id read_id, - bool contents) +static int ima_read_file(struct file *file, enum kernel_read_file_id read_id, + bool contents) { enum ima_hooks func; u32 secid; @@ -801,8 +801,8 @@ const int read_idmap[READING_MAX_ID] = { * On success return 0. On integrity appraisal error, assuming the file * is in policy and IMA-appraisal is in enforcing mode, return -EACCES. */ -int ima_post_read_file(struct file *file, void *buf, loff_t size, - enum kernel_read_file_id read_id) +static int ima_post_read_file(struct file *file, char *buf, loff_t size, + enum kernel_read_file_id read_id) { enum ima_hooks func; u32 secid; @@ -835,7 +835,7 @@ int ima_post_read_file(struct file *file, void *buf, loff_t size, * * For permission return 0, otherwise return -EACCES. */ -int ima_load_data(enum kernel_load_data_id id, bool contents) +static int ima_load_data(enum kernel_load_data_id id, bool contents) { bool ima_enforce, sig_enforce; @@ -889,9 +889,9 @@ int ima_load_data(enum kernel_load_data_id id, bool contents) * On success return 0. On integrity appraisal error, assuming the file * is in policy and IMA-appraisal is in enforcing mode, return -EACCES. */ -int ima_post_load_data(char *buf, loff_t size, - enum kernel_load_data_id load_id, - char *description) +static int ima_post_load_data(char *buf, loff_t size, + enum kernel_load_data_id load_id, + char *description) { if (load_id == LOADING_FIRMWARE) { if ((ima_appraise & IMA_APPRAISE_FIRMWARE) && @@ -934,7 +934,7 @@ int process_buffer_measurement(struct mnt_idmap *idmap, int ret = 0; const char *audit_cause = "ENOMEM"; struct ima_template_entry *entry = NULL; - struct integrity_iint_cache iint = {}; + struct ima_iint_cache iint = {}; struct ima_event_data event_data = {.iint = &iint, .filename = eventname, .buf = buf, @@ -1089,6 +1089,39 @@ int ima_measure_critical_data(const char *event_label, } EXPORT_SYMBOL_GPL(ima_measure_critical_data); +#ifdef CONFIG_INTEGRITY_ASYMMETRIC_KEYS + +/** + * ima_kernel_module_request - Prevent crypto-pkcs1pad(rsa,*) requests + * @kmod_name: kernel module name + * + * Avoid a verification loop where verifying the signature of the modprobe + * binary requires executing modprobe itself. Since the modprobe iint->mutex + * is already held when the signature verification is performed, a deadlock + * occurs as soon as modprobe is executed within the critical region, since + * the same lock cannot be taken again. + * + * This happens when public_key_verify_signature(), in case of RSA algorithm, + * use alg_name to store internal information in order to construct an + * algorithm on the fly, but crypto_larval_lookup() will try to use alg_name + * in order to load a kernel module with same name. + * + * Since we don't have any real "crypto-pkcs1pad(rsa,*)" kernel modules, + * we are safe to fail such module request from crypto_larval_lookup(), and + * avoid the verification loop. + * + * Return: Zero if it is safe to load the kernel module, -EINVAL otherwise. + */ +static int ima_kernel_module_request(char *kmod_name) +{ + if (strncmp(kmod_name, "crypto-pkcs1pad(rsa,", 20) == 0) + return -EINVAL; + + return 0; +} + +#endif /* CONFIG_INTEGRITY_ASYMMETRIC_KEYS */ + static int __init init_ima(void) { int error; @@ -1120,4 +1153,49 @@ static int __init init_ima(void) return error; } +static struct security_hook_list ima_hooks[] __ro_after_init = { + LSM_HOOK_INIT(bprm_check_security, ima_bprm_check), + LSM_HOOK_INIT(file_post_open, ima_file_check), + LSM_HOOK_INIT(inode_post_create_tmpfile, ima_post_create_tmpfile), + LSM_HOOK_INIT(file_release, ima_file_free), + LSM_HOOK_INIT(mmap_file, ima_file_mmap), + LSM_HOOK_INIT(file_mprotect, ima_file_mprotect), + LSM_HOOK_INIT(kernel_load_data, ima_load_data), + LSM_HOOK_INIT(kernel_post_load_data, ima_post_load_data), + LSM_HOOK_INIT(kernel_read_file, ima_read_file), + LSM_HOOK_INIT(kernel_post_read_file, ima_post_read_file), + LSM_HOOK_INIT(path_post_mknod, ima_post_path_mknod), +#ifdef CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS + LSM_HOOK_INIT(key_post_create_or_update, ima_post_key_create_or_update), +#endif +#ifdef CONFIG_INTEGRITY_ASYMMETRIC_KEYS + LSM_HOOK_INIT(kernel_module_request, ima_kernel_module_request), +#endif + LSM_HOOK_INIT(inode_free_security, ima_inode_free), +}; + +static const struct lsm_id ima_lsmid = { + .name = "ima", + .id = LSM_ID_IMA, +}; + +static int __init init_ima_lsm(void) +{ + ima_iintcache_init(); + security_add_hooks(ima_hooks, ARRAY_SIZE(ima_hooks), &ima_lsmid); + init_ima_appraise_lsm(&ima_lsmid); + return 0; +} + +struct lsm_blob_sizes ima_blob_sizes __ro_after_init = { + .lbs_inode = sizeof(struct ima_iint_cache *), +}; + +DEFINE_LSM(ima) = { + .name = "ima", + .init = init_ima_lsm, + .order = LSM_ORDER_LAST, + .blobs = &ima_blob_sizes, +}; + late_initcall(init_ima); /* Start IMA after the TPM is available */ diff --git a/security/integrity/ima/ima_policy.c b/security/integrity/ima/ima_policy.c index f69062617754..c0556907c2e6 100644 --- a/security/integrity/ima/ima_policy.c +++ b/security/integrity/ima/ima_policy.c @@ -49,7 +49,7 @@ #define DONT_HASH 0x0200 #define INVALID_PCR(a) (((a) < 0) || \ - (a) >= (sizeof_field(struct integrity_iint_cache, measured_pcrs) * 8)) + (a) >= (sizeof_field(struct ima_iint_cache, measured_pcrs) * 8)) int ima_policy_flag; static int temp_ima_appraise; diff --git a/security/integrity/integrity.h b/security/integrity/integrity.h index 9561db7cf6b4..50d6f798e613 100644 --- a/security/integrity/integrity.h +++ b/security/integrity/integrity.h @@ -18,60 +18,7 @@ #include #include #include - -/* iint action cache flags */ -#define IMA_MEASURE 0x00000001 -#define IMA_MEASURED 0x00000002 -#define IMA_APPRAISE 0x00000004 -#define IMA_APPRAISED 0x00000008 -/*#define IMA_COLLECT 0x00000010 do not use this flag */ -#define IMA_COLLECTED 0x00000020 -#define IMA_AUDIT 0x00000040 -#define IMA_AUDITED 0x00000080 -#define IMA_HASH 0x00000100 -#define IMA_HASHED 0x00000200 - -/* iint policy rule cache flags */ -#define IMA_NONACTION_FLAGS 0xff000000 -#define IMA_DIGSIG_REQUIRED 0x01000000 -#define IMA_PERMIT_DIRECTIO 0x02000000 -#define IMA_NEW_FILE 0x04000000 -#define EVM_IMMUTABLE_DIGSIG 0x08000000 -#define IMA_FAIL_UNVERIFIABLE_SIGS 0x10000000 -#define IMA_MODSIG_ALLOWED 0x20000000 -#define IMA_CHECK_BLACKLIST 0x40000000 -#define IMA_VERITY_REQUIRED 0x80000000 - -#define IMA_DO_MASK (IMA_MEASURE | IMA_APPRAISE | IMA_AUDIT | \ - IMA_HASH | IMA_APPRAISE_SUBMASK) -#define IMA_DONE_MASK (IMA_MEASURED | IMA_APPRAISED | IMA_AUDITED | \ - IMA_HASHED | IMA_COLLECTED | \ - IMA_APPRAISED_SUBMASK) - -/* iint subaction appraise cache flags */ -#define IMA_FILE_APPRAISE 0x00001000 -#define IMA_FILE_APPRAISED 0x00002000 -#define IMA_MMAP_APPRAISE 0x00004000 -#define IMA_MMAP_APPRAISED 0x00008000 -#define IMA_BPRM_APPRAISE 0x00010000 -#define IMA_BPRM_APPRAISED 0x00020000 -#define IMA_READ_APPRAISE 0x00040000 -#define IMA_READ_APPRAISED 0x00080000 -#define IMA_CREDS_APPRAISE 0x00100000 -#define IMA_CREDS_APPRAISED 0x00200000 -#define IMA_APPRAISE_SUBMASK (IMA_FILE_APPRAISE | IMA_MMAP_APPRAISE | \ - IMA_BPRM_APPRAISE | IMA_READ_APPRAISE | \ - IMA_CREDS_APPRAISE) -#define IMA_APPRAISED_SUBMASK (IMA_FILE_APPRAISED | IMA_MMAP_APPRAISED | \ - IMA_BPRM_APPRAISED | IMA_READ_APPRAISED | \ - IMA_CREDS_APPRAISED) - -/* iint cache atomic_flags */ -#define IMA_CHANGE_XATTR 0 -#define IMA_UPDATE_XATTR 1 -#define IMA_CHANGE_ATTR 2 -#define IMA_DIGSIG 3 -#define IMA_MUST_MEASURE 4 +#include enum evm_ima_xattr_type { IMA_XATTR_DIGEST = 0x01, @@ -155,31 +102,6 @@ struct ima_file_id { __u8 hash[HASH_MAX_DIGESTSIZE]; } __packed; -/* integrity data associated with an inode */ -struct integrity_iint_cache { - struct rb_node rb_node; /* rooted in integrity_iint_tree */ - struct mutex mutex; /* protects: version, flags, digest */ - struct inode *inode; /* back pointer to inode in question */ - u64 version; /* track inode changes */ - unsigned long flags; - unsigned long measured_pcrs; - unsigned long atomic_flags; - unsigned long real_ino; - dev_t real_dev; - enum integrity_status ima_file_status:4; - enum integrity_status ima_mmap_status:4; - enum integrity_status ima_bprm_status:4; - enum integrity_status ima_read_status:4; - enum integrity_status ima_creds_status:4; - enum integrity_status evm_status:4; - struct ima_digest_data *ima_hash; -}; - -/* rbtree tree calls to lookup, insert, delete - * integrity data associated with an inode. - */ -struct integrity_iint_cache *integrity_iint_find(struct inode *inode); - int integrity_kernel_read(struct file *file, loff_t offset, void *addr, unsigned long count); diff --git a/security/keys/key.c b/security/keys/key.c index 5b10641debd5..560790038329 100644 --- a/security/keys/key.c +++ b/security/keys/key.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include "internal.h" @@ -930,8 +929,8 @@ static key_ref_t __key_create_or_update(key_ref_t keyring_ref, goto error_link_end; } - ima_post_key_create_or_update(keyring, key, payload, plen, - flags, true); + security_key_post_create_or_update(keyring, key, payload, plen, flags, + true); key_ref = make_key_ref(key, is_key_possessed(keyring_ref)); @@ -964,9 +963,8 @@ error: key_ref = __key_update(key_ref, &prep); if (!IS_ERR(key_ref)) - ima_post_key_create_or_update(keyring, key, - payload, plen, - flags, false); + security_key_post_create_or_update(keyring, key, payload, plen, + flags, false); goto error_free_prep; } diff --git a/security/security.c b/security/security.c index aef69632d0a9..a07f43c58d65 100644 --- a/security/security.c +++ b/security/security.c @@ -19,15 +19,13 @@ #include #include #include -#include -#include -#include #include #include #include #include #include #include +#include #include #include #include @@ -51,7 +49,9 @@ (IS_ENABLED(CONFIG_SECURITY_SAFESETID) ? 1 : 0) + \ (IS_ENABLED(CONFIG_SECURITY_LOCKDOWN_LSM) ? 1 : 0) + \ (IS_ENABLED(CONFIG_BPF_LSM) ? 1 : 0) + \ - (IS_ENABLED(CONFIG_SECURITY_LANDLOCK) ? 1 : 0)) + (IS_ENABLED(CONFIG_SECURITY_LANDLOCK) ? 1 : 0) + \ + (IS_ENABLED(CONFIG_IMA) ? 1 : 0) + \ + (IS_ENABLED(CONFIG_EVM) ? 1 : 0)) /* * These are descriptions of the reasons that can be passed to the @@ -856,14 +856,14 @@ out: P->hook.FUNC(__VA_ARGS__); \ } while (0) -#define call_int_hook(FUNC, IRC, ...) ({ \ - int RC = IRC; \ +#define call_int_hook(FUNC, ...) ({ \ + int RC = LSM_RET_DEFAULT(FUNC); \ do { \ struct security_hook_list *P; \ \ hlist_for_each_entry(P, &security_hook_heads.FUNC, list) { \ RC = P->hook.FUNC(__VA_ARGS__); \ - if (RC != 0) \ + if (RC != LSM_RET_DEFAULT(FUNC)) \ break; \ } \ } while (0); \ @@ -882,7 +882,7 @@ out: */ int security_binder_set_context_mgr(const struct cred *mgr) { - return call_int_hook(binder_set_context_mgr, 0, mgr); + return call_int_hook(binder_set_context_mgr, mgr); } /** @@ -897,7 +897,7 @@ int security_binder_set_context_mgr(const struct cred *mgr) int security_binder_transaction(const struct cred *from, const struct cred *to) { - return call_int_hook(binder_transaction, 0, from, to); + return call_int_hook(binder_transaction, from, to); } /** @@ -912,7 +912,7 @@ int security_binder_transaction(const struct cred *from, int security_binder_transfer_binder(const struct cred *from, const struct cred *to) { - return call_int_hook(binder_transfer_binder, 0, from, to); + return call_int_hook(binder_transfer_binder, from, to); } /** @@ -928,7 +928,7 @@ int security_binder_transfer_binder(const struct cred *from, int security_binder_transfer_file(const struct cred *from, const struct cred *to, const struct file *file) { - return call_int_hook(binder_transfer_file, 0, from, to, file); + return call_int_hook(binder_transfer_file, from, to, file); } /** @@ -947,7 +947,7 @@ int security_binder_transfer_file(const struct cred *from, */ int security_ptrace_access_check(struct task_struct *child, unsigned int mode) { - return call_int_hook(ptrace_access_check, 0, child, mode); + return call_int_hook(ptrace_access_check, child, mode); } /** @@ -962,7 +962,7 @@ int security_ptrace_access_check(struct task_struct *child, unsigned int mode) */ int security_ptrace_traceme(struct task_struct *parent) { - return call_int_hook(ptrace_traceme, 0, parent); + return call_int_hook(ptrace_traceme, parent); } /** @@ -984,8 +984,7 @@ int security_capget(const struct task_struct *target, kernel_cap_t *inheritable, kernel_cap_t *permitted) { - return call_int_hook(capget, 0, target, - effective, inheritable, permitted); + return call_int_hook(capget, target, effective, inheritable, permitted); } /** @@ -1006,8 +1005,8 @@ int security_capset(struct cred *new, const struct cred *old, const kernel_cap_t *inheritable, const kernel_cap_t *permitted) { - return call_int_hook(capset, 0, new, old, - effective, inheritable, permitted); + return call_int_hook(capset, new, old, effective, inheritable, + permitted); } /** @@ -1028,7 +1027,7 @@ int security_capable(const struct cred *cred, int cap, unsigned int opts) { - return call_int_hook(capable, 0, cred, ns, cap, opts); + return call_int_hook(capable, cred, ns, cap, opts); } /** @@ -1044,7 +1043,7 @@ int security_capable(const struct cred *cred, */ int security_quotactl(int cmds, int type, int id, const struct super_block *sb) { - return call_int_hook(quotactl, 0, cmds, type, id, sb); + return call_int_hook(quotactl, cmds, type, id, sb); } /** @@ -1057,7 +1056,7 @@ int security_quotactl(int cmds, int type, int id, const struct super_block *sb) */ int security_quota_on(struct dentry *dentry) { - return call_int_hook(quota_on, 0, dentry); + return call_int_hook(quota_on, dentry); } /** @@ -1072,7 +1071,7 @@ int security_quota_on(struct dentry *dentry) */ int security_syslog(int type) { - return call_int_hook(syslog, 0, type); + return call_int_hook(syslog, type); } /** @@ -1087,7 +1086,7 @@ int security_syslog(int type) */ int security_settime64(const struct timespec64 *ts, const struct timezone *tz) { - return call_int_hook(settime, 0, ts, tz); + return call_int_hook(settime, ts, tz); } /** @@ -1142,7 +1141,7 @@ int security_vm_enough_memory_mm(struct mm_struct *mm, long pages) */ int security_bprm_creds_for_exec(struct linux_binprm *bprm) { - return call_int_hook(bprm_creds_for_exec, 0, bprm); + return call_int_hook(bprm_creds_for_exec, bprm); } /** @@ -1166,7 +1165,7 @@ int security_bprm_creds_for_exec(struct linux_binprm *bprm) */ int security_bprm_creds_from_file(struct linux_binprm *bprm, const struct file *file) { - return call_int_hook(bprm_creds_from_file, 0, bprm, file); + return call_int_hook(bprm_creds_from_file, bprm, file); } /** @@ -1183,12 +1182,7 @@ int security_bprm_creds_from_file(struct linux_binprm *bprm, const struct file * */ int security_bprm_check(struct linux_binprm *bprm) { - int ret; - - ret = call_int_hook(bprm_check_security, 0, bprm); - if (ret) - return ret; - return ima_bprm_check(bprm); + return call_int_hook(bprm_check_security, bprm); } /** @@ -1235,7 +1229,7 @@ void security_bprm_committed_creds(const struct linux_binprm *bprm) */ int security_fs_context_submount(struct fs_context *fc, struct super_block *reference) { - return call_int_hook(fs_context_submount, 0, fc, reference); + return call_int_hook(fs_context_submount, fc, reference); } /** @@ -1251,7 +1245,7 @@ int security_fs_context_submount(struct fs_context *fc, struct super_block *refe */ int security_fs_context_dup(struct fs_context *fc, struct fs_context *src_fc) { - return call_int_hook(fs_context_dup, 0, fc, src_fc); + return call_int_hook(fs_context_dup, fc, src_fc); } /** @@ -1300,7 +1294,7 @@ int security_sb_alloc(struct super_block *sb) if (unlikely(rc)) return rc; - rc = call_int_hook(sb_alloc_security, 0, sb); + rc = call_int_hook(sb_alloc_security, sb); if (unlikely(rc)) security_sb_free(sb); return rc; @@ -1358,7 +1352,7 @@ EXPORT_SYMBOL(security_free_mnt_opts); */ int security_sb_eat_lsm_opts(char *options, void **mnt_opts) { - return call_int_hook(sb_eat_lsm_opts, 0, options, mnt_opts); + return call_int_hook(sb_eat_lsm_opts, options, mnt_opts); } EXPORT_SYMBOL(security_sb_eat_lsm_opts); @@ -1375,7 +1369,7 @@ EXPORT_SYMBOL(security_sb_eat_lsm_opts); int security_sb_mnt_opts_compat(struct super_block *sb, void *mnt_opts) { - return call_int_hook(sb_mnt_opts_compat, 0, sb, mnt_opts); + return call_int_hook(sb_mnt_opts_compat, sb, mnt_opts); } EXPORT_SYMBOL(security_sb_mnt_opts_compat); @@ -1392,7 +1386,7 @@ EXPORT_SYMBOL(security_sb_mnt_opts_compat); int security_sb_remount(struct super_block *sb, void *mnt_opts) { - return call_int_hook(sb_remount, 0, sb, mnt_opts); + return call_int_hook(sb_remount, sb, mnt_opts); } EXPORT_SYMBOL(security_sb_remount); @@ -1406,7 +1400,7 @@ EXPORT_SYMBOL(security_sb_remount); */ int security_sb_kern_mount(const struct super_block *sb) { - return call_int_hook(sb_kern_mount, 0, sb); + return call_int_hook(sb_kern_mount, sb); } /** @@ -1420,7 +1414,7 @@ int security_sb_kern_mount(const struct super_block *sb) */ int security_sb_show_options(struct seq_file *m, struct super_block *sb) { - return call_int_hook(sb_show_options, 0, m, sb); + return call_int_hook(sb_show_options, m, sb); } /** @@ -1434,7 +1428,7 @@ int security_sb_show_options(struct seq_file *m, struct super_block *sb) */ int security_sb_statfs(struct dentry *dentry) { - return call_int_hook(sb_statfs, 0, dentry); + return call_int_hook(sb_statfs, dentry); } /** @@ -1457,7 +1451,7 @@ int security_sb_statfs(struct dentry *dentry) int security_sb_mount(const char *dev_name, const struct path *path, const char *type, unsigned long flags, void *data) { - return call_int_hook(sb_mount, 0, dev_name, path, type, flags, data); + return call_int_hook(sb_mount, dev_name, path, type, flags, data); } /** @@ -1471,7 +1465,7 @@ int security_sb_mount(const char *dev_name, const struct path *path, */ int security_sb_umount(struct vfsmount *mnt, int flags) { - return call_int_hook(sb_umount, 0, mnt, flags); + return call_int_hook(sb_umount, mnt, flags); } /** @@ -1486,7 +1480,7 @@ int security_sb_umount(struct vfsmount *mnt, int flags) int security_sb_pivotroot(const struct path *old_path, const struct path *new_path) { - return call_int_hook(sb_pivotroot, 0, old_path, new_path); + return call_int_hook(sb_pivotroot, old_path, new_path); } /** @@ -1505,9 +1499,17 @@ int security_sb_set_mnt_opts(struct super_block *sb, unsigned long kern_flags, unsigned long *set_kern_flags) { - return call_int_hook(sb_set_mnt_opts, - mnt_opts ? -EOPNOTSUPP : 0, sb, - mnt_opts, kern_flags, set_kern_flags); + struct security_hook_list *hp; + int rc = mnt_opts ? -EOPNOTSUPP : LSM_RET_DEFAULT(sb_set_mnt_opts); + + hlist_for_each_entry(hp, &security_hook_heads.sb_set_mnt_opts, + list) { + rc = hp->hook.sb_set_mnt_opts(sb, mnt_opts, kern_flags, + set_kern_flags); + if (rc != LSM_RET_DEFAULT(sb_set_mnt_opts)) + break; + } + return rc; } EXPORT_SYMBOL(security_sb_set_mnt_opts); @@ -1527,7 +1529,7 @@ int security_sb_clone_mnt_opts(const struct super_block *oldsb, unsigned long kern_flags, unsigned long *set_kern_flags) { - return call_int_hook(sb_clone_mnt_opts, 0, oldsb, newsb, + return call_int_hook(sb_clone_mnt_opts, oldsb, newsb, kern_flags, set_kern_flags); } EXPORT_SYMBOL(security_sb_clone_mnt_opts); @@ -1544,7 +1546,7 @@ EXPORT_SYMBOL(security_sb_clone_mnt_opts); int security_move_mount(const struct path *from_path, const struct path *to_path) { - return call_int_hook(move_mount, 0, from_path, to_path); + return call_int_hook(move_mount, from_path, to_path); } /** @@ -1561,7 +1563,7 @@ int security_move_mount(const struct path *from_path, int security_path_notify(const struct path *path, u64 mask, unsigned int obj_type) { - return call_int_hook(path_notify, 0, path, mask, obj_type); + return call_int_hook(path_notify, path, mask, obj_type); } /** @@ -1580,7 +1582,7 @@ int security_inode_alloc(struct inode *inode) if (unlikely(rc)) return rc; - rc = call_int_hook(inode_alloc_security, 0, inode); + rc = call_int_hook(inode_alloc_security, inode); if (unlikely(rc)) security_inode_free(inode); return rc; @@ -1602,7 +1604,6 @@ static void inode_free_by_rcu(struct rcu_head *head) */ void security_inode_free(struct inode *inode) { - integrity_inode_free(inode); call_void_hook(inode_free_security, inode); /* * The inode may still be referenced in a path walk and @@ -1638,20 +1639,8 @@ int security_dentry_init_security(struct dentry *dentry, int mode, const char **xattr_name, void **ctx, u32 *ctxlen) { - struct security_hook_list *hp; - int rc; - - /* - * Only one module will provide a security context. - */ - hlist_for_each_entry(hp, &security_hook_heads.dentry_init_security, - list) { - rc = hp->hook.dentry_init_security(dentry, mode, name, - xattr_name, ctx, ctxlen); - if (rc != LSM_RET_DEFAULT(dentry_init_security)) - return rc; - } - return LSM_RET_DEFAULT(dentry_init_security); + return call_int_hook(dentry_init_security, dentry, mode, name, + xattr_name, ctx, ctxlen); } EXPORT_SYMBOL(security_dentry_init_security); @@ -1674,7 +1663,7 @@ int security_dentry_create_files_as(struct dentry *dentry, int mode, struct qstr *name, const struct cred *old, struct cred *new) { - return call_int_hook(dentry_create_files_as, 0, dentry, mode, + return call_int_hook(dentry_create_files_as, dentry, mode, name, old, new); } EXPORT_SYMBOL(security_dentry_create_files_as); @@ -1721,8 +1710,8 @@ int security_inode_init_security(struct inode *inode, struct inode *dir, return 0; if (initxattrs) { - /* Allocate +1 for EVM and +1 as terminator. */ - new_xattrs = kcalloc(blob_sizes.lbs_xattr_count + 2, + /* Allocate +1 as terminator. */ + new_xattrs = kcalloc(blob_sizes.lbs_xattr_count + 1, sizeof(*new_xattrs), GFP_NOFS); if (!new_xattrs) return -ENOMEM; @@ -1746,10 +1735,6 @@ int security_inode_init_security(struct inode *inode, struct inode *dir, if (!xattr_count) goto out; - ret = evm_inode_init_security(inode, dir, qstr, new_xattrs, - &xattr_count); - if (ret) - goto out; ret = initxattrs(inode, new_xattrs, fs_data); out: for (; xattr_count > 0; xattr_count--) @@ -1775,7 +1760,7 @@ int security_inode_init_security_anon(struct inode *inode, const struct qstr *name, const struct inode *context_inode) { - return call_int_hook(inode_init_security_anon, 0, inode, name, + return call_int_hook(inode_init_security_anon, inode, name, context_inode); } @@ -1797,10 +1782,24 @@ int security_path_mknod(const struct path *dir, struct dentry *dentry, { if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry)))) return 0; - return call_int_hook(path_mknod, 0, dir, dentry, mode, dev); + return call_int_hook(path_mknod, dir, dentry, mode, dev); } EXPORT_SYMBOL(security_path_mknod); +/** + * security_path_post_mknod() - Update inode security field after file creation + * @idmap: idmap of the mount + * @dentry: new file + * + * Update inode security field after a file has been created. + */ +void security_path_post_mknod(struct mnt_idmap *idmap, struct dentry *dentry) +{ + if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) + return; + call_void_hook(path_post_mknod, idmap, dentry); +} + /** * security_path_mkdir() - Check if creating a new directory is allowed * @dir: parent directory @@ -1816,7 +1815,7 @@ int security_path_mkdir(const struct path *dir, struct dentry *dentry, { if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry)))) return 0; - return call_int_hook(path_mkdir, 0, dir, dentry, mode); + return call_int_hook(path_mkdir, dir, dentry, mode); } EXPORT_SYMBOL(security_path_mkdir); @@ -1833,7 +1832,7 @@ int security_path_rmdir(const struct path *dir, struct dentry *dentry) { if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry)))) return 0; - return call_int_hook(path_rmdir, 0, dir, dentry); + return call_int_hook(path_rmdir, dir, dentry); } /** @@ -1849,7 +1848,7 @@ int security_path_unlink(const struct path *dir, struct dentry *dentry) { if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry)))) return 0; - return call_int_hook(path_unlink, 0, dir, dentry); + return call_int_hook(path_unlink, dir, dentry); } EXPORT_SYMBOL(security_path_unlink); @@ -1868,7 +1867,7 @@ int security_path_symlink(const struct path *dir, struct dentry *dentry, { if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry)))) return 0; - return call_int_hook(path_symlink, 0, dir, dentry, old_name); + return call_int_hook(path_symlink, dir, dentry, old_name); } /** @@ -1886,7 +1885,7 @@ int security_path_link(struct dentry *old_dentry, const struct path *new_dir, { if (unlikely(IS_PRIVATE(d_backing_inode(old_dentry)))) return 0; - return call_int_hook(path_link, 0, old_dentry, new_dir, new_dentry); + return call_int_hook(path_link, old_dentry, new_dir, new_dentry); } /** @@ -1910,7 +1909,7 @@ int security_path_rename(const struct path *old_dir, struct dentry *old_dentry, IS_PRIVATE(d_backing_inode(new_dentry))))) return 0; - return call_int_hook(path_rename, 0, old_dir, old_dentry, new_dir, + return call_int_hook(path_rename, old_dir, old_dentry, new_dir, new_dentry, flags); } EXPORT_SYMBOL(security_path_rename); @@ -1929,7 +1928,7 @@ int security_path_truncate(const struct path *path) { if (unlikely(IS_PRIVATE(d_backing_inode(path->dentry)))) return 0; - return call_int_hook(path_truncate, 0, path); + return call_int_hook(path_truncate, path); } /** @@ -1947,7 +1946,7 @@ int security_path_chmod(const struct path *path, umode_t mode) { if (unlikely(IS_PRIVATE(d_backing_inode(path->dentry)))) return 0; - return call_int_hook(path_chmod, 0, path, mode); + return call_int_hook(path_chmod, path, mode); } /** @@ -1964,7 +1963,7 @@ int security_path_chown(const struct path *path, kuid_t uid, kgid_t gid) { if (unlikely(IS_PRIVATE(d_backing_inode(path->dentry)))) return 0; - return call_int_hook(path_chown, 0, path, uid, gid); + return call_int_hook(path_chown, path, uid, gid); } /** @@ -1977,7 +1976,7 @@ int security_path_chown(const struct path *path, kuid_t uid, kgid_t gid) */ int security_path_chroot(const struct path *path) { - return call_int_hook(path_chroot, 0, path); + return call_int_hook(path_chroot, path); } #endif /* CONFIG_SECURITY_PATH */ @@ -1996,10 +1995,25 @@ int security_inode_create(struct inode *dir, struct dentry *dentry, { if (unlikely(IS_PRIVATE(dir))) return 0; - return call_int_hook(inode_create, 0, dir, dentry, mode); + return call_int_hook(inode_create, dir, dentry, mode); } EXPORT_SYMBOL_GPL(security_inode_create); +/** + * security_inode_post_create_tmpfile() - Update inode security of new tmpfile + * @idmap: idmap of the mount + * @inode: inode of the new tmpfile + * + * Update inode security data after a tmpfile has been created. + */ +void security_inode_post_create_tmpfile(struct mnt_idmap *idmap, + struct inode *inode) +{ + if (unlikely(IS_PRIVATE(inode))) + return; + call_void_hook(inode_post_create_tmpfile, idmap, inode); +} + /** * security_inode_link() - Check if creating a hard link is allowed * @old_dentry: existing file @@ -2015,7 +2029,7 @@ int security_inode_link(struct dentry *old_dentry, struct inode *dir, { if (unlikely(IS_PRIVATE(d_backing_inode(old_dentry)))) return 0; - return call_int_hook(inode_link, 0, old_dentry, dir, new_dentry); + return call_int_hook(inode_link, old_dentry, dir, new_dentry); } /** @@ -2031,7 +2045,7 @@ int security_inode_unlink(struct inode *dir, struct dentry *dentry) { if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - return call_int_hook(inode_unlink, 0, dir, dentry); + return call_int_hook(inode_unlink, dir, dentry); } /** @@ -2049,7 +2063,7 @@ int security_inode_symlink(struct inode *dir, struct dentry *dentry, { if (unlikely(IS_PRIVATE(dir))) return 0; - return call_int_hook(inode_symlink, 0, dir, dentry, old_name); + return call_int_hook(inode_symlink, dir, dentry, old_name); } /** @@ -2067,7 +2081,7 @@ int security_inode_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) { if (unlikely(IS_PRIVATE(dir))) return 0; - return call_int_hook(inode_mkdir, 0, dir, dentry, mode); + return call_int_hook(inode_mkdir, dir, dentry, mode); } EXPORT_SYMBOL_GPL(security_inode_mkdir); @@ -2084,7 +2098,7 @@ int security_inode_rmdir(struct inode *dir, struct dentry *dentry) { if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - return call_int_hook(inode_rmdir, 0, dir, dentry); + return call_int_hook(inode_rmdir, dir, dentry); } /** @@ -2106,7 +2120,7 @@ int security_inode_mknod(struct inode *dir, struct dentry *dentry, { if (unlikely(IS_PRIVATE(dir))) return 0; - return call_int_hook(inode_mknod, 0, dir, dentry, mode, dev); + return call_int_hook(inode_mknod, dir, dentry, mode, dev); } /** @@ -2131,13 +2145,13 @@ int security_inode_rename(struct inode *old_dir, struct dentry *old_dentry, return 0; if (flags & RENAME_EXCHANGE) { - int err = call_int_hook(inode_rename, 0, new_dir, new_dentry, + int err = call_int_hook(inode_rename, new_dir, new_dentry, old_dir, old_dentry); if (err) return err; } - return call_int_hook(inode_rename, 0, old_dir, old_dentry, + return call_int_hook(inode_rename, old_dir, old_dentry, new_dir, new_dentry); } @@ -2153,7 +2167,7 @@ int security_inode_readlink(struct dentry *dentry) { if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - return call_int_hook(inode_readlink, 0, dentry); + return call_int_hook(inode_readlink, dentry); } /** @@ -2172,7 +2186,7 @@ int security_inode_follow_link(struct dentry *dentry, struct inode *inode, { if (unlikely(IS_PRIVATE(inode))) return 0; - return call_int_hook(inode_follow_link, 0, dentry, inode, rcu); + return call_int_hook(inode_follow_link, dentry, inode, rcu); } /** @@ -2193,7 +2207,7 @@ int security_inode_permission(struct inode *inode, int mask) { if (unlikely(IS_PRIVATE(inode))) return 0; - return call_int_hook(inode_permission, 0, inode, mask); + return call_int_hook(inode_permission, inode, mask); } /** @@ -2212,17 +2226,28 @@ int security_inode_permission(struct inode *inode, int mask) int security_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr) { - int ret; - if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - ret = call_int_hook(inode_setattr, 0, dentry, attr); - if (ret) - return ret; - return evm_inode_setattr(idmap, dentry, attr); + return call_int_hook(inode_setattr, idmap, dentry, attr); } EXPORT_SYMBOL_GPL(security_inode_setattr); +/** + * security_inode_post_setattr() - Update the inode after a setattr operation + * @idmap: idmap of the mount + * @dentry: file + * @ia_valid: file attributes set + * + * Update inode security field after successful setting file attributes. + */ +void security_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + int ia_valid) +{ + if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) + return; + call_void_hook(inode_post_setattr, idmap, dentry, ia_valid); +} + /** * security_inode_getattr() - Check if getting file attributes is allowed * @path: file @@ -2235,7 +2260,7 @@ int security_inode_getattr(const struct path *path) { if (unlikely(IS_PRIVATE(d_backing_inode(path->dentry)))) return 0; - return call_int_hook(inode_getattr, 0, path); + return call_int_hook(inode_getattr, path); } /** @@ -2263,17 +2288,12 @@ int security_inode_setxattr(struct mnt_idmap *idmap, * SELinux and Smack integrate the cap call, * so assume that all LSMs supplying this call do so. */ - ret = call_int_hook(inode_setxattr, 1, idmap, dentry, name, value, - size, flags); + ret = call_int_hook(inode_setxattr, idmap, dentry, name, value, size, + flags); if (ret == 1) ret = cap_inode_setxattr(dentry, name, value, size, flags); - if (ret) - return ret; - ret = ima_inode_setxattr(dentry, name, value, size); - if (ret) - return ret; - return evm_inode_setxattr(idmap, dentry, name, value, size); + return ret; } /** @@ -2292,18 +2312,26 @@ int security_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl) { - int ret; - if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - ret = call_int_hook(inode_set_acl, 0, idmap, dentry, acl_name, - kacl); - if (ret) - return ret; - ret = ima_inode_set_acl(idmap, dentry, acl_name, kacl); - if (ret) - return ret; - return evm_inode_set_acl(idmap, dentry, acl_name, kacl); + return call_int_hook(inode_set_acl, idmap, dentry, acl_name, kacl); +} + +/** + * security_inode_post_set_acl() - Update inode security from posix acls set + * @dentry: file + * @acl_name: acl name + * @kacl: acl struct + * + * Update inode security data after successfully setting posix acls on @dentry. + * The posix acls in @kacl are identified by @acl_name. + */ +void security_inode_post_set_acl(struct dentry *dentry, const char *acl_name, + struct posix_acl *kacl) +{ + if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) + return; + call_void_hook(inode_post_set_acl, dentry, acl_name, kacl); } /** @@ -2322,7 +2350,7 @@ int security_inode_get_acl(struct mnt_idmap *idmap, { if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - return call_int_hook(inode_get_acl, 0, idmap, dentry, acl_name); + return call_int_hook(inode_get_acl, idmap, dentry, acl_name); } /** @@ -2339,17 +2367,26 @@ int security_inode_get_acl(struct mnt_idmap *idmap, int security_inode_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) { - int ret; - if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - ret = call_int_hook(inode_remove_acl, 0, idmap, dentry, acl_name); - if (ret) - return ret; - ret = ima_inode_remove_acl(idmap, dentry, acl_name); - if (ret) - return ret; - return evm_inode_remove_acl(idmap, dentry, acl_name); + return call_int_hook(inode_remove_acl, idmap, dentry, acl_name); +} + +/** + * security_inode_post_remove_acl() - Update inode security after rm posix acls + * @idmap: idmap of the mount + * @dentry: file + * @acl_name: acl name + * + * Update inode security data after successfully removing posix acls on + * @dentry in @idmap. The posix acls are identified by @acl_name. + */ +void security_inode_post_remove_acl(struct mnt_idmap *idmap, + struct dentry *dentry, const char *acl_name) +{ + if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) + return; + call_void_hook(inode_post_remove_acl, idmap, dentry, acl_name); } /** @@ -2368,7 +2405,6 @@ void security_inode_post_setxattr(struct dentry *dentry, const char *name, if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return; call_void_hook(inode_post_setxattr, dentry, name, value, size, flags); - evm_inode_post_setxattr(dentry, name, value, size); } /** @@ -2385,7 +2421,7 @@ int security_inode_getxattr(struct dentry *dentry, const char *name) { if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - return call_int_hook(inode_getxattr, 0, dentry, name); + return call_int_hook(inode_getxattr, dentry, name); } /** @@ -2401,7 +2437,7 @@ int security_inode_listxattr(struct dentry *dentry) { if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) return 0; - return call_int_hook(inode_listxattr, 0, dentry); + return call_int_hook(inode_listxattr, dentry); } /** @@ -2426,15 +2462,24 @@ int security_inode_removexattr(struct mnt_idmap *idmap, * SELinux and Smack integrate the cap call, * so assume that all LSMs supplying this call do so. */ - ret = call_int_hook(inode_removexattr, 1, idmap, dentry, name); + ret = call_int_hook(inode_removexattr, idmap, dentry, name); if (ret == 1) ret = cap_inode_removexattr(idmap, dentry, name); - if (ret) - return ret; - ret = ima_inode_removexattr(dentry, name); - if (ret) - return ret; - return evm_inode_removexattr(idmap, dentry, name); + return ret; +} + +/** + * security_inode_post_removexattr() - Update the inode after a removexattr op + * @dentry: file + * @name: xattr name + * + * Update the inode after a successful removexattr operation. + */ +void security_inode_post_removexattr(struct dentry *dentry, const char *name) +{ + if (unlikely(IS_PRIVATE(d_backing_inode(dentry)))) + return; + call_void_hook(inode_post_removexattr, dentry, name); } /** @@ -2450,7 +2495,7 @@ int security_inode_removexattr(struct mnt_idmap *idmap, */ int security_inode_need_killpriv(struct dentry *dentry) { - return call_int_hook(inode_need_killpriv, 0, dentry); + return call_int_hook(inode_need_killpriv, dentry); } /** @@ -2467,7 +2512,7 @@ int security_inode_need_killpriv(struct dentry *dentry) int security_inode_killpriv(struct mnt_idmap *idmap, struct dentry *dentry) { - return call_int_hook(inode_killpriv, 0, idmap, dentry); + return call_int_hook(inode_killpriv, idmap, dentry); } /** @@ -2490,21 +2535,11 @@ int security_inode_getsecurity(struct mnt_idmap *idmap, struct inode *inode, const char *name, void **buffer, bool alloc) { - struct security_hook_list *hp; - int rc; - if (unlikely(IS_PRIVATE(inode))) return LSM_RET_DEFAULT(inode_getsecurity); - /* - * Only one module will provide an attribute with a given name. - */ - hlist_for_each_entry(hp, &security_hook_heads.inode_getsecurity, list) { - rc = hp->hook.inode_getsecurity(idmap, inode, name, buffer, - alloc); - if (rc != LSM_RET_DEFAULT(inode_getsecurity)) - return rc; - } - return LSM_RET_DEFAULT(inode_getsecurity); + + return call_int_hook(inode_getsecurity, idmap, inode, name, buffer, + alloc); } /** @@ -2525,21 +2560,11 @@ int security_inode_getsecurity(struct mnt_idmap *idmap, int security_inode_setsecurity(struct inode *inode, const char *name, const void *value, size_t size, int flags) { - struct security_hook_list *hp; - int rc; - if (unlikely(IS_PRIVATE(inode))) return LSM_RET_DEFAULT(inode_setsecurity); - /* - * Only one module will provide an attribute with a given name. - */ - hlist_for_each_entry(hp, &security_hook_heads.inode_setsecurity, list) { - rc = hp->hook.inode_setsecurity(inode, name, value, size, - flags); - if (rc != LSM_RET_DEFAULT(inode_setsecurity)) - return rc; - } - return LSM_RET_DEFAULT(inode_setsecurity); + + return call_int_hook(inode_setsecurity, inode, name, value, size, + flags); } /** @@ -2560,7 +2585,7 @@ int security_inode_listsecurity(struct inode *inode, { if (unlikely(IS_PRIVATE(inode))) return 0; - return call_int_hook(inode_listsecurity, 0, inode, buffer, buffer_size); + return call_int_hook(inode_listsecurity, inode, buffer, buffer_size); } EXPORT_SYMBOL(security_inode_listsecurity); @@ -2591,7 +2616,7 @@ void security_inode_getsecid(struct inode *inode, u32 *secid) */ int security_inode_copy_up(struct dentry *src, struct cred **new) { - return call_int_hook(inode_copy_up, 0, src, new); + return call_int_hook(inode_copy_up, src, new); } EXPORT_SYMBOL(security_inode_copy_up); @@ -2609,7 +2634,6 @@ EXPORT_SYMBOL(security_inode_copy_up); */ int security_inode_copy_up_xattr(const char *name) { - struct security_hook_list *hp; int rc; /* @@ -2617,14 +2641,11 @@ int security_inode_copy_up_xattr(const char *name) * xattr), -EOPNOTSUPP if it does not know anything about the xattr or * any other error code in case of an error. */ - hlist_for_each_entry(hp, - &security_hook_heads.inode_copy_up_xattr, list) { - rc = hp->hook.inode_copy_up_xattr(name); - if (rc != LSM_RET_DEFAULT(inode_copy_up_xattr)) - return rc; - } + rc = call_int_hook(inode_copy_up_xattr, name); + if (rc != LSM_RET_DEFAULT(inode_copy_up_xattr)) + return rc; - return evm_inode_copy_up_xattr(name); + return LSM_RET_DEFAULT(inode_copy_up_xattr); } EXPORT_SYMBOL(security_inode_copy_up_xattr); @@ -2641,7 +2662,7 @@ EXPORT_SYMBOL(security_inode_copy_up_xattr); int security_kernfs_init_security(struct kernfs_node *kn_dir, struct kernfs_node *kn) { - return call_int_hook(kernfs_init_security, 0, kn_dir, kn); + return call_int_hook(kernfs_init_security, kn_dir, kn); } /** @@ -2665,7 +2686,7 @@ int security_kernfs_init_security(struct kernfs_node *kn_dir, */ int security_file_permission(struct file *file, int mask) { - return call_int_hook(file_permission, 0, file, mask); + return call_int_hook(file_permission, file, mask); } /** @@ -2683,12 +2704,23 @@ int security_file_alloc(struct file *file) if (rc) return rc; - rc = call_int_hook(file_alloc_security, 0, file); + rc = call_int_hook(file_alloc_security, file); if (unlikely(rc)) security_file_free(file); return rc; } +/** + * security_file_release() - Perform actions before releasing the file ref + * @file: the file + * + * Perform actions before releasing the last reference to a file. + */ +void security_file_release(struct file *file) +{ + call_void_hook(file_release, file); +} + /** * security_file_free() - Free a file's LSM blob * @file: the file @@ -2723,7 +2755,7 @@ void security_file_free(struct file *file) */ int security_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { - return call_int_hook(file_ioctl, 0, file, cmd, arg); + return call_int_hook(file_ioctl, file, cmd, arg); } EXPORT_SYMBOL_GPL(security_file_ioctl); @@ -2741,7 +2773,7 @@ EXPORT_SYMBOL_GPL(security_file_ioctl); int security_file_ioctl_compat(struct file *file, unsigned int cmd, unsigned long arg) { - return call_int_hook(file_ioctl_compat, 0, file, cmd, arg); + return call_int_hook(file_ioctl_compat, file, cmd, arg); } EXPORT_SYMBOL_GPL(security_file_ioctl_compat); @@ -2792,13 +2824,8 @@ static inline unsigned long mmap_prot(struct file *file, unsigned long prot) int security_mmap_file(struct file *file, unsigned long prot, unsigned long flags) { - unsigned long prot_adj = mmap_prot(file, prot); - int ret; - - ret = call_int_hook(mmap_file, 0, file, prot, prot_adj, flags); - if (ret) - return ret; - return ima_file_mmap(file, prot, prot_adj, flags); + return call_int_hook(mmap_file, file, prot, mmap_prot(file, prot), + flags); } /** @@ -2811,7 +2838,7 @@ int security_mmap_file(struct file *file, unsigned long prot, */ int security_mmap_addr(unsigned long addr) { - return call_int_hook(mmap_addr, 0, addr); + return call_int_hook(mmap_addr, addr); } /** @@ -2827,12 +2854,7 @@ int security_mmap_addr(unsigned long addr) int security_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, unsigned long prot) { - int ret; - - ret = call_int_hook(file_mprotect, 0, vma, reqprot, prot); - if (ret) - return ret; - return ima_file_mprotect(vma, prot); + return call_int_hook(file_mprotect, vma, reqprot, prot); } /** @@ -2847,7 +2869,7 @@ int security_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, */ int security_file_lock(struct file *file, unsigned int cmd) { - return call_int_hook(file_lock, 0, file, cmd); + return call_int_hook(file_lock, file, cmd); } /** @@ -2866,7 +2888,7 @@ int security_file_lock(struct file *file, unsigned int cmd) */ int security_file_fcntl(struct file *file, unsigned int cmd, unsigned long arg) { - return call_int_hook(file_fcntl, 0, file, cmd, arg); + return call_int_hook(file_fcntl, file, cmd, arg); } /** @@ -2900,11 +2922,11 @@ void security_file_set_fowner(struct file *file) int security_file_send_sigiotask(struct task_struct *tsk, struct fown_struct *fown, int sig) { - return call_int_hook(file_send_sigiotask, 0, tsk, fown, sig); + return call_int_hook(file_send_sigiotask, tsk, fown, sig); } /** - * security_file_receive() - Check is receiving a file via IPC is allowed + * security_file_receive() - Check if receiving a file via IPC is allowed * @file: file being received * * This hook allows security modules to control the ability of a process to @@ -2914,7 +2936,7 @@ int security_file_send_sigiotask(struct task_struct *tsk, */ int security_file_receive(struct file *file) { - return call_int_hook(file_receive, 0, file); + return call_int_hook(file_receive, file); } /** @@ -2930,13 +2952,30 @@ int security_file_open(struct file *file) { int ret; - ret = call_int_hook(file_open, 0, file); + ret = call_int_hook(file_open, file); if (ret) return ret; return fsnotify_open_perm(file); } +/** + * security_file_post_open() - Evaluate a file after it has been opened + * @file: the file + * @mask: access mask + * + * Evaluate an opened file and the access mask requested with open(). The hook + * is useful for LSMs that require the file content to be available in order to + * make decisions. + * + * Return: Returns 0 if permission is granted. + */ +int security_file_post_open(struct file *file, int mask) +{ + return call_int_hook(file_post_open, file, mask); +} +EXPORT_SYMBOL_GPL(security_file_post_open); + /** * security_file_truncate() - Check if truncating a file is allowed * @file: file @@ -2949,7 +2988,7 @@ int security_file_open(struct file *file) */ int security_file_truncate(struct file *file) { - return call_int_hook(file_truncate, 0, file); + return call_int_hook(file_truncate, file); } /** @@ -2967,7 +3006,7 @@ int security_task_alloc(struct task_struct *task, unsigned long clone_flags) if (rc) return rc; - rc = call_int_hook(task_alloc, 0, task, clone_flags); + rc = call_int_hook(task_alloc, task, clone_flags); if (unlikely(rc)) security_task_free(task); return rc; @@ -3005,7 +3044,7 @@ int security_cred_alloc_blank(struct cred *cred, gfp_t gfp) if (rc) return rc; - rc = call_int_hook(cred_alloc_blank, 0, cred, gfp); + rc = call_int_hook(cred_alloc_blank, cred, gfp); if (unlikely(rc)) security_cred_free(cred); return rc; @@ -3049,7 +3088,7 @@ int security_prepare_creds(struct cred *new, const struct cred *old, gfp_t gfp) if (rc) return rc; - rc = call_int_hook(cred_prepare, 0, new, old, gfp); + rc = call_int_hook(cred_prepare, new, old, gfp); if (unlikely(rc)) security_cred_free(new); return rc; @@ -3094,7 +3133,7 @@ EXPORT_SYMBOL(security_cred_getsecid); */ int security_kernel_act_as(struct cred *new, u32 secid) { - return call_int_hook(kernel_act_as, 0, new, secid); + return call_int_hook(kernel_act_as, new, secid); } /** @@ -3110,11 +3149,11 @@ int security_kernel_act_as(struct cred *new, u32 secid) */ int security_kernel_create_files_as(struct cred *new, struct inode *inode) { - return call_int_hook(kernel_create_files_as, 0, new, inode); + return call_int_hook(kernel_create_files_as, new, inode); } /** - * security_kernel_module_request() - Check is loading a module is allowed + * security_kernel_module_request() - Check if loading a module is allowed * @kmod_name: module name * * Ability to trigger the kernel to automatically upcall to userspace for @@ -3124,12 +3163,7 @@ int security_kernel_create_files_as(struct cred *new, struct inode *inode) */ int security_kernel_module_request(char *kmod_name) { - int ret; - - ret = call_int_hook(kernel_module_request, 0, kmod_name); - if (ret) - return ret; - return integrity_kernel_module_request(kmod_name); + return call_int_hook(kernel_module_request, kmod_name); } /** @@ -3145,12 +3179,7 @@ int security_kernel_module_request(char *kmod_name) int security_kernel_read_file(struct file *file, enum kernel_read_file_id id, bool contents) { - int ret; - - ret = call_int_hook(kernel_read_file, 0, file, id, contents); - if (ret) - return ret; - return ima_read_file(file, id, contents); + return call_int_hook(kernel_read_file, file, id, contents); } EXPORT_SYMBOL_GPL(security_kernel_read_file); @@ -3170,12 +3199,7 @@ EXPORT_SYMBOL_GPL(security_kernel_read_file); int security_kernel_post_read_file(struct file *file, char *buf, loff_t size, enum kernel_read_file_id id) { - int ret; - - ret = call_int_hook(kernel_post_read_file, 0, file, buf, size, id); - if (ret) - return ret; - return ima_post_read_file(file, buf, size, id); + return call_int_hook(kernel_post_read_file, file, buf, size, id); } EXPORT_SYMBOL_GPL(security_kernel_post_read_file); @@ -3190,12 +3214,7 @@ EXPORT_SYMBOL_GPL(security_kernel_post_read_file); */ int security_kernel_load_data(enum kernel_load_data_id id, bool contents) { - int ret; - - ret = call_int_hook(kernel_load_data, 0, id, contents); - if (ret) - return ret; - return ima_load_data(id, contents); + return call_int_hook(kernel_load_data, id, contents); } EXPORT_SYMBOL_GPL(security_kernel_load_data); @@ -3217,13 +3236,7 @@ int security_kernel_post_load_data(char *buf, loff_t size, enum kernel_load_data_id id, char *description) { - int ret; - - ret = call_int_hook(kernel_post_load_data, 0, buf, size, id, - description); - if (ret) - return ret; - return ima_post_load_data(buf, size, id, description); + return call_int_hook(kernel_post_load_data, buf, size, id, description); } EXPORT_SYMBOL_GPL(security_kernel_post_load_data); @@ -3244,7 +3257,7 @@ EXPORT_SYMBOL_GPL(security_kernel_post_load_data); int security_task_fix_setuid(struct cred *new, const struct cred *old, int flags) { - return call_int_hook(task_fix_setuid, 0, new, old, flags); + return call_int_hook(task_fix_setuid, new, old, flags); } /** @@ -3264,7 +3277,7 @@ int security_task_fix_setuid(struct cred *new, const struct cred *old, int security_task_fix_setgid(struct cred *new, const struct cred *old, int flags) { - return call_int_hook(task_fix_setgid, 0, new, old, flags); + return call_int_hook(task_fix_setgid, new, old, flags); } /** @@ -3281,7 +3294,7 @@ int security_task_fix_setgid(struct cred *new, const struct cred *old, */ int security_task_fix_setgroups(struct cred *new, const struct cred *old) { - return call_int_hook(task_fix_setgroups, 0, new, old); + return call_int_hook(task_fix_setgroups, new, old); } /** @@ -3296,7 +3309,7 @@ int security_task_fix_setgroups(struct cred *new, const struct cred *old) */ int security_task_setpgid(struct task_struct *p, pid_t pgid) { - return call_int_hook(task_setpgid, 0, p, pgid); + return call_int_hook(task_setpgid, p, pgid); } /** @@ -3310,7 +3323,7 @@ int security_task_setpgid(struct task_struct *p, pid_t pgid) */ int security_task_getpgid(struct task_struct *p) { - return call_int_hook(task_getpgid, 0, p); + return call_int_hook(task_getpgid, p); } /** @@ -3323,7 +3336,7 @@ int security_task_getpgid(struct task_struct *p) */ int security_task_getsid(struct task_struct *p) { - return call_int_hook(task_getsid, 0, p); + return call_int_hook(task_getsid, p); } /** @@ -3366,7 +3379,7 @@ EXPORT_SYMBOL(security_task_getsecid_obj); */ int security_task_setnice(struct task_struct *p, int nice) { - return call_int_hook(task_setnice, 0, p, nice); + return call_int_hook(task_setnice, p, nice); } /** @@ -3380,7 +3393,7 @@ int security_task_setnice(struct task_struct *p, int nice) */ int security_task_setioprio(struct task_struct *p, int ioprio) { - return call_int_hook(task_setioprio, 0, p, ioprio); + return call_int_hook(task_setioprio, p, ioprio); } /** @@ -3393,7 +3406,7 @@ int security_task_setioprio(struct task_struct *p, int ioprio) */ int security_task_getioprio(struct task_struct *p) { - return call_int_hook(task_getioprio, 0, p); + return call_int_hook(task_getioprio, p); } /** @@ -3410,7 +3423,7 @@ int security_task_getioprio(struct task_struct *p) int security_task_prlimit(const struct cred *cred, const struct cred *tcred, unsigned int flags) { - return call_int_hook(task_prlimit, 0, cred, tcred, flags); + return call_int_hook(task_prlimit, cred, tcred, flags); } /** @@ -3428,7 +3441,7 @@ int security_task_prlimit(const struct cred *cred, const struct cred *tcred, int security_task_setrlimit(struct task_struct *p, unsigned int resource, struct rlimit *new_rlim) { - return call_int_hook(task_setrlimit, 0, p, resource, new_rlim); + return call_int_hook(task_setrlimit, p, resource, new_rlim); } /** @@ -3442,7 +3455,7 @@ int security_task_setrlimit(struct task_struct *p, unsigned int resource, */ int security_task_setscheduler(struct task_struct *p) { - return call_int_hook(task_setscheduler, 0, p); + return call_int_hook(task_setscheduler, p); } /** @@ -3455,7 +3468,7 @@ int security_task_setscheduler(struct task_struct *p) */ int security_task_getscheduler(struct task_struct *p) { - return call_int_hook(task_getscheduler, 0, p); + return call_int_hook(task_getscheduler, p); } /** @@ -3468,7 +3481,7 @@ int security_task_getscheduler(struct task_struct *p) */ int security_task_movememory(struct task_struct *p) { - return call_int_hook(task_movememory, 0, p); + return call_int_hook(task_movememory, p); } /** @@ -3489,7 +3502,7 @@ int security_task_movememory(struct task_struct *p) int security_task_kill(struct task_struct *p, struct kernel_siginfo *info, int sig, const struct cred *cred) { - return call_int_hook(task_kill, 0, p, info, sig, cred); + return call_int_hook(task_kill, p, info, sig, cred); } /** @@ -3547,7 +3560,7 @@ void security_task_to_inode(struct task_struct *p, struct inode *inode) */ int security_create_user_ns(const struct cred *cred) { - return call_int_hook(userns_create, 0, cred); + return call_int_hook(userns_create, cred); } /** @@ -3561,7 +3574,7 @@ int security_create_user_ns(const struct cred *cred) */ int security_ipc_permission(struct kern_ipc_perm *ipcp, short flag) { - return call_int_hook(ipc_permission, 0, ipcp, flag); + return call_int_hook(ipc_permission, ipcp, flag); } /** @@ -3593,7 +3606,7 @@ int security_msg_msg_alloc(struct msg_msg *msg) if (unlikely(rc)) return rc; - rc = call_int_hook(msg_msg_alloc_security, 0, msg); + rc = call_int_hook(msg_msg_alloc_security, msg); if (unlikely(rc)) security_msg_msg_free(msg); return rc; @@ -3627,7 +3640,7 @@ int security_msg_queue_alloc(struct kern_ipc_perm *msq) if (unlikely(rc)) return rc; - rc = call_int_hook(msg_queue_alloc_security, 0, msq); + rc = call_int_hook(msg_queue_alloc_security, msq); if (unlikely(rc)) security_msg_queue_free(msq); return rc; @@ -3659,7 +3672,7 @@ void security_msg_queue_free(struct kern_ipc_perm *msq) */ int security_msg_queue_associate(struct kern_ipc_perm *msq, int msqflg) { - return call_int_hook(msg_queue_associate, 0, msq, msqflg); + return call_int_hook(msg_queue_associate, msq, msqflg); } /** @@ -3674,7 +3687,7 @@ int security_msg_queue_associate(struct kern_ipc_perm *msq, int msqflg) */ int security_msg_queue_msgctl(struct kern_ipc_perm *msq, int cmd) { - return call_int_hook(msg_queue_msgctl, 0, msq, cmd); + return call_int_hook(msg_queue_msgctl, msq, cmd); } /** @@ -3691,7 +3704,7 @@ int security_msg_queue_msgctl(struct kern_ipc_perm *msq, int cmd) int security_msg_queue_msgsnd(struct kern_ipc_perm *msq, struct msg_msg *msg, int msqflg) { - return call_int_hook(msg_queue_msgsnd, 0, msq, msg, msqflg); + return call_int_hook(msg_queue_msgsnd, msq, msg, msqflg); } /** @@ -3712,7 +3725,7 @@ int security_msg_queue_msgsnd(struct kern_ipc_perm *msq, int security_msg_queue_msgrcv(struct kern_ipc_perm *msq, struct msg_msg *msg, struct task_struct *target, long type, int mode) { - return call_int_hook(msg_queue_msgrcv, 0, msq, msg, target, type, mode); + return call_int_hook(msg_queue_msgrcv, msq, msg, target, type, mode); } /** @@ -3730,7 +3743,7 @@ int security_shm_alloc(struct kern_ipc_perm *shp) if (unlikely(rc)) return rc; - rc = call_int_hook(shm_alloc_security, 0, shp); + rc = call_int_hook(shm_alloc_security, shp); if (unlikely(rc)) security_shm_free(shp); return rc; @@ -3763,7 +3776,7 @@ void security_shm_free(struct kern_ipc_perm *shp) */ int security_shm_associate(struct kern_ipc_perm *shp, int shmflg) { - return call_int_hook(shm_associate, 0, shp, shmflg); + return call_int_hook(shm_associate, shp, shmflg); } /** @@ -3778,7 +3791,7 @@ int security_shm_associate(struct kern_ipc_perm *shp, int shmflg) */ int security_shm_shmctl(struct kern_ipc_perm *shp, int cmd) { - return call_int_hook(shm_shmctl, 0, shp, cmd); + return call_int_hook(shm_shmctl, shp, cmd); } /** @@ -3796,7 +3809,7 @@ int security_shm_shmctl(struct kern_ipc_perm *shp, int cmd) int security_shm_shmat(struct kern_ipc_perm *shp, char __user *shmaddr, int shmflg) { - return call_int_hook(shm_shmat, 0, shp, shmaddr, shmflg); + return call_int_hook(shm_shmat, shp, shmaddr, shmflg); } /** @@ -3814,7 +3827,7 @@ int security_sem_alloc(struct kern_ipc_perm *sma) if (unlikely(rc)) return rc; - rc = call_int_hook(sem_alloc_security, 0, sma); + rc = call_int_hook(sem_alloc_security, sma); if (unlikely(rc)) security_sem_free(sma); return rc; @@ -3846,7 +3859,7 @@ void security_sem_free(struct kern_ipc_perm *sma) */ int security_sem_associate(struct kern_ipc_perm *sma, int semflg) { - return call_int_hook(sem_associate, 0, sma, semflg); + return call_int_hook(sem_associate, sma, semflg); } /** @@ -3861,7 +3874,7 @@ int security_sem_associate(struct kern_ipc_perm *sma, int semflg) */ int security_sem_semctl(struct kern_ipc_perm *sma, int cmd) { - return call_int_hook(sem_semctl, 0, sma, cmd); + return call_int_hook(sem_semctl, sma, cmd); } /** @@ -3879,7 +3892,7 @@ int security_sem_semctl(struct kern_ipc_perm *sma, int cmd) int security_sem_semop(struct kern_ipc_perm *sma, struct sembuf *sops, unsigned nsops, int alter) { - return call_int_hook(sem_semop, 0, sma, sops, nsops, alter); + return call_int_hook(sem_semop, sma, sops, nsops, alter); } /** @@ -4110,11 +4123,11 @@ int security_setprocattr(int lsmid, const char *name, void *value, size_t size) */ int security_netlink_send(struct sock *sk, struct sk_buff *skb) { - return call_int_hook(netlink_send, 0, sk, skb); + return call_int_hook(netlink_send, sk, skb); } /** - * security_ismaclabel() - Check is the named attribute is a MAC label + * security_ismaclabel() - Check if the named attribute is a MAC label * @name: full extended attribute name * * Check if the extended attribute specified by @name represents a MAC label. @@ -4123,7 +4136,7 @@ int security_netlink_send(struct sock *sk, struct sk_buff *skb) */ int security_ismaclabel(const char *name) { - return call_int_hook(ismaclabel, 0, name); + return call_int_hook(ismaclabel, name); } EXPORT_SYMBOL(security_ismaclabel); @@ -4142,20 +4155,7 @@ EXPORT_SYMBOL(security_ismaclabel); */ int security_secid_to_secctx(u32 secid, char **secdata, u32 *seclen) { - struct security_hook_list *hp; - int rc; - - /* - * Currently, only one LSM can implement secid_to_secctx (i.e this - * LSM hook is not "stackable"). - */ - hlist_for_each_entry(hp, &security_hook_heads.secid_to_secctx, list) { - rc = hp->hook.secid_to_secctx(secid, secdata, seclen); - if (rc != LSM_RET_DEFAULT(secid_to_secctx)) - return rc; - } - - return LSM_RET_DEFAULT(secid_to_secctx); + return call_int_hook(secid_to_secctx, secid, secdata, seclen); } EXPORT_SYMBOL(security_secid_to_secctx); @@ -4172,7 +4172,7 @@ EXPORT_SYMBOL(security_secid_to_secctx); int security_secctx_to_secid(const char *secdata, u32 seclen, u32 *secid) { *secid = 0; - return call_int_hook(secctx_to_secid, 0, secdata, seclen, secid); + return call_int_hook(secctx_to_secid, secdata, seclen, secid); } EXPORT_SYMBOL(security_secctx_to_secid); @@ -4219,7 +4219,7 @@ EXPORT_SYMBOL(security_inode_invalidate_secctx); */ int security_inode_notifysecctx(struct inode *inode, void *ctx, u32 ctxlen) { - return call_int_hook(inode_notifysecctx, 0, inode, ctx, ctxlen); + return call_int_hook(inode_notifysecctx, inode, ctx, ctxlen); } EXPORT_SYMBOL(security_inode_notifysecctx); @@ -4241,7 +4241,7 @@ EXPORT_SYMBOL(security_inode_notifysecctx); */ int security_inode_setsecctx(struct dentry *dentry, void *ctx, u32 ctxlen) { - return call_int_hook(inode_setsecctx, 0, dentry, ctx, ctxlen); + return call_int_hook(inode_setsecctx, dentry, ctx, ctxlen); } EXPORT_SYMBOL(security_inode_setsecctx); @@ -4258,19 +4258,7 @@ EXPORT_SYMBOL(security_inode_setsecctx); */ int security_inode_getsecctx(struct inode *inode, void **ctx, u32 *ctxlen) { - struct security_hook_list *hp; - int rc; - - /* - * Only one module will provide a security context. - */ - hlist_for_each_entry(hp, &security_hook_heads.inode_getsecctx, list) { - rc = hp->hook.inode_getsecctx(inode, ctx, ctxlen); - if (rc != LSM_RET_DEFAULT(inode_getsecctx)) - return rc; - } - - return LSM_RET_DEFAULT(inode_getsecctx); + return call_int_hook(inode_getsecctx, inode, ctx, ctxlen); } EXPORT_SYMBOL(security_inode_getsecctx); @@ -4289,7 +4277,7 @@ int security_post_notification(const struct cred *w_cred, const struct cred *cred, struct watch_notification *n) { - return call_int_hook(post_notification, 0, w_cred, cred, n); + return call_int_hook(post_notification, w_cred, cred, n); } #endif /* CONFIG_WATCH_QUEUE */ @@ -4305,7 +4293,7 @@ int security_post_notification(const struct cred *w_cred, */ int security_watch_key(struct key *key) { - return call_int_hook(watch_key, 0, key); + return call_int_hook(watch_key, key); } #endif /* CONFIG_KEY_NOTIFICATIONS */ @@ -4334,7 +4322,7 @@ int security_watch_key(struct key *key) int security_unix_stream_connect(struct sock *sock, struct sock *other, struct sock *newsk) { - return call_int_hook(unix_stream_connect, 0, sock, other, newsk); + return call_int_hook(unix_stream_connect, sock, other, newsk); } EXPORT_SYMBOL(security_unix_stream_connect); @@ -4360,7 +4348,7 @@ EXPORT_SYMBOL(security_unix_stream_connect); */ int security_unix_may_send(struct socket *sock, struct socket *other) { - return call_int_hook(unix_may_send, 0, sock, other); + return call_int_hook(unix_may_send, sock, other); } EXPORT_SYMBOL(security_unix_may_send); @@ -4377,7 +4365,7 @@ EXPORT_SYMBOL(security_unix_may_send); */ int security_socket_create(int family, int type, int protocol, int kern) { - return call_int_hook(socket_create, 0, family, type, protocol, kern); + return call_int_hook(socket_create, family, type, protocol, kern); } /** @@ -4401,7 +4389,7 @@ int security_socket_create(int family, int type, int protocol, int kern) int security_socket_post_create(struct socket *sock, int family, int type, int protocol, int kern) { - return call_int_hook(socket_post_create, 0, sock, family, type, + return call_int_hook(socket_post_create, sock, family, type, protocol, kern); } @@ -4417,7 +4405,7 @@ int security_socket_post_create(struct socket *sock, int family, */ int security_socket_socketpair(struct socket *socka, struct socket *sockb) { - return call_int_hook(socket_socketpair, 0, socka, sockb); + return call_int_hook(socket_socketpair, socka, sockb); } EXPORT_SYMBOL(security_socket_socketpair); @@ -4436,7 +4424,7 @@ EXPORT_SYMBOL(security_socket_socketpair); int security_socket_bind(struct socket *sock, struct sockaddr *address, int addrlen) { - return call_int_hook(socket_bind, 0, sock, address, addrlen); + return call_int_hook(socket_bind, sock, address, addrlen); } /** @@ -4453,7 +4441,7 @@ int security_socket_bind(struct socket *sock, int security_socket_connect(struct socket *sock, struct sockaddr *address, int addrlen) { - return call_int_hook(socket_connect, 0, sock, address, addrlen); + return call_int_hook(socket_connect, sock, address, addrlen); } /** @@ -4467,7 +4455,7 @@ int security_socket_connect(struct socket *sock, */ int security_socket_listen(struct socket *sock, int backlog) { - return call_int_hook(socket_listen, 0, sock, backlog); + return call_int_hook(socket_listen, sock, backlog); } /** @@ -4483,11 +4471,11 @@ int security_socket_listen(struct socket *sock, int backlog) */ int security_socket_accept(struct socket *sock, struct socket *newsock) { - return call_int_hook(socket_accept, 0, sock, newsock); + return call_int_hook(socket_accept, sock, newsock); } /** - * security_socket_sendmsg() - Check is sending a message is allowed + * security_socket_sendmsg() - Check if sending a message is allowed * @sock: sending socket * @msg: message to send * @size: size of message @@ -4498,7 +4486,7 @@ int security_socket_accept(struct socket *sock, struct socket *newsock) */ int security_socket_sendmsg(struct socket *sock, struct msghdr *msg, int size) { - return call_int_hook(socket_sendmsg, 0, sock, msg, size); + return call_int_hook(socket_sendmsg, sock, msg, size); } /** @@ -4515,7 +4503,7 @@ int security_socket_sendmsg(struct socket *sock, struct msghdr *msg, int size) int security_socket_recvmsg(struct socket *sock, struct msghdr *msg, int size, int flags) { - return call_int_hook(socket_recvmsg, 0, sock, msg, size, flags); + return call_int_hook(socket_recvmsg, sock, msg, size, flags); } /** @@ -4529,7 +4517,7 @@ int security_socket_recvmsg(struct socket *sock, struct msghdr *msg, */ int security_socket_getsockname(struct socket *sock) { - return call_int_hook(socket_getsockname, 0, sock); + return call_int_hook(socket_getsockname, sock); } /** @@ -4542,7 +4530,7 @@ int security_socket_getsockname(struct socket *sock) */ int security_socket_getpeername(struct socket *sock) { - return call_int_hook(socket_getpeername, 0, sock); + return call_int_hook(socket_getpeername, sock); } /** @@ -4558,7 +4546,7 @@ int security_socket_getpeername(struct socket *sock) */ int security_socket_getsockopt(struct socket *sock, int level, int optname) { - return call_int_hook(socket_getsockopt, 0, sock, level, optname); + return call_int_hook(socket_getsockopt, sock, level, optname); } /** @@ -4573,7 +4561,7 @@ int security_socket_getsockopt(struct socket *sock, int level, int optname) */ int security_socket_setsockopt(struct socket *sock, int level, int optname) { - return call_int_hook(socket_setsockopt, 0, sock, level, optname); + return call_int_hook(socket_setsockopt, sock, level, optname); } /** @@ -4588,7 +4576,7 @@ int security_socket_setsockopt(struct socket *sock, int level, int optname) */ int security_socket_shutdown(struct socket *sock, int how) { - return call_int_hook(socket_shutdown, 0, sock, how); + return call_int_hook(socket_shutdown, sock, how); } /** @@ -4605,7 +4593,7 @@ int security_socket_shutdown(struct socket *sock, int how) */ int security_sock_rcv_skb(struct sock *sk, struct sk_buff *skb) { - return call_int_hook(socket_sock_rcv_skb, 0, sk, skb); + return call_int_hook(socket_sock_rcv_skb, sk, skb); } EXPORT_SYMBOL(security_sock_rcv_skb); @@ -4627,20 +4615,8 @@ EXPORT_SYMBOL(security_sock_rcv_skb); int security_socket_getpeersec_stream(struct socket *sock, sockptr_t optval, sockptr_t optlen, unsigned int len) { - struct security_hook_list *hp; - int rc; - - /* - * Only one module will provide a security context. - */ - hlist_for_each_entry(hp, &security_hook_heads.socket_getpeersec_stream, - list) { - rc = hp->hook.socket_getpeersec_stream(sock, optval, optlen, - len); - if (rc != LSM_RET_DEFAULT(socket_getpeersec_stream)) - return rc; - } - return LSM_RET_DEFAULT(socket_getpeersec_stream); + return call_int_hook(socket_getpeersec_stream, sock, optval, optlen, + len); } /** @@ -4660,19 +4636,7 @@ int security_socket_getpeersec_stream(struct socket *sock, sockptr_t optval, int security_socket_getpeersec_dgram(struct socket *sock, struct sk_buff *skb, u32 *secid) { - struct security_hook_list *hp; - int rc; - - /* - * Only one module will provide a security context. - */ - hlist_for_each_entry(hp, &security_hook_heads.socket_getpeersec_dgram, - list) { - rc = hp->hook.socket_getpeersec_dgram(sock, skb, secid); - if (rc != LSM_RET_DEFAULT(socket_getpeersec_dgram)) - return rc; - } - return LSM_RET_DEFAULT(socket_getpeersec_dgram); + return call_int_hook(socket_getpeersec_dgram, sock, skb, secid); } EXPORT_SYMBOL(security_socket_getpeersec_dgram); @@ -4689,7 +4653,7 @@ EXPORT_SYMBOL(security_socket_getpeersec_dgram); */ int security_sk_alloc(struct sock *sk, int family, gfp_t priority) { - return call_int_hook(sk_alloc_security, 0, sk, family, priority); + return call_int_hook(sk_alloc_security, sk, family, priority); } /** @@ -4770,7 +4734,7 @@ EXPORT_SYMBOL(security_sock_graft); int security_inet_conn_request(const struct sock *sk, struct sk_buff *skb, struct request_sock *req) { - return call_int_hook(inet_conn_request, 0, sk, skb, req); + return call_int_hook(inet_conn_request, sk, skb, req); } EXPORT_SYMBOL(security_inet_conn_request); @@ -4811,7 +4775,7 @@ EXPORT_SYMBOL(security_inet_conn_established); */ int security_secmark_relabel_packet(u32 secid) { - return call_int_hook(secmark_relabel_packet, 0, secid); + return call_int_hook(secmark_relabel_packet, secid); } EXPORT_SYMBOL(security_secmark_relabel_packet); @@ -4848,7 +4812,7 @@ EXPORT_SYMBOL(security_secmark_refcount_dec); */ int security_tun_dev_alloc_security(void **security) { - return call_int_hook(tun_dev_alloc_security, 0, security); + return call_int_hook(tun_dev_alloc_security, security); } EXPORT_SYMBOL(security_tun_dev_alloc_security); @@ -4873,7 +4837,7 @@ EXPORT_SYMBOL(security_tun_dev_free_security); */ int security_tun_dev_create(void) { - return call_int_hook(tun_dev_create, 0); + return call_int_hook(tun_dev_create); } EXPORT_SYMBOL(security_tun_dev_create); @@ -4887,7 +4851,7 @@ EXPORT_SYMBOL(security_tun_dev_create); */ int security_tun_dev_attach_queue(void *security) { - return call_int_hook(tun_dev_attach_queue, 0, security); + return call_int_hook(tun_dev_attach_queue, security); } EXPORT_SYMBOL(security_tun_dev_attach_queue); @@ -4903,7 +4867,7 @@ EXPORT_SYMBOL(security_tun_dev_attach_queue); */ int security_tun_dev_attach(struct sock *sk, void *security) { - return call_int_hook(tun_dev_attach, 0, sk, security); + return call_int_hook(tun_dev_attach, sk, security); } EXPORT_SYMBOL(security_tun_dev_attach); @@ -4918,7 +4882,7 @@ EXPORT_SYMBOL(security_tun_dev_attach); */ int security_tun_dev_open(void *security) { - return call_int_hook(tun_dev_open, 0, security); + return call_int_hook(tun_dev_open, security); } EXPORT_SYMBOL(security_tun_dev_open); @@ -4934,7 +4898,7 @@ EXPORT_SYMBOL(security_tun_dev_open); int security_sctp_assoc_request(struct sctp_association *asoc, struct sk_buff *skb) { - return call_int_hook(sctp_assoc_request, 0, asoc, skb); + return call_int_hook(sctp_assoc_request, asoc, skb); } EXPORT_SYMBOL(security_sctp_assoc_request); @@ -4955,8 +4919,7 @@ EXPORT_SYMBOL(security_sctp_assoc_request); int security_sctp_bind_connect(struct sock *sk, int optname, struct sockaddr *address, int addrlen) { - return call_int_hook(sctp_bind_connect, 0, sk, optname, - address, addrlen); + return call_int_hook(sctp_bind_connect, sk, optname, address, addrlen); } EXPORT_SYMBOL(security_sctp_bind_connect); @@ -4990,7 +4953,7 @@ EXPORT_SYMBOL(security_sctp_sk_clone); int security_sctp_assoc_established(struct sctp_association *asoc, struct sk_buff *skb) { - return call_int_hook(sctp_assoc_established, 0, asoc, skb); + return call_int_hook(sctp_assoc_established, asoc, skb); } EXPORT_SYMBOL(security_sctp_assoc_established); @@ -5008,7 +4971,7 @@ EXPORT_SYMBOL(security_sctp_assoc_established); */ int security_mptcp_add_subflow(struct sock *sk, struct sock *ssk) { - return call_int_hook(mptcp_add_subflow, 0, sk, ssk); + return call_int_hook(mptcp_add_subflow, sk, ssk); } #endif /* CONFIG_SECURITY_NETWORK */ @@ -5026,7 +4989,7 @@ int security_mptcp_add_subflow(struct sock *sk, struct sock *ssk) */ int security_ib_pkey_access(void *sec, u64 subnet_prefix, u16 pkey) { - return call_int_hook(ib_pkey_access, 0, sec, subnet_prefix, pkey); + return call_int_hook(ib_pkey_access, sec, subnet_prefix, pkey); } EXPORT_SYMBOL(security_ib_pkey_access); @@ -5043,8 +5006,7 @@ EXPORT_SYMBOL(security_ib_pkey_access); int security_ib_endport_manage_subnet(void *sec, const char *dev_name, u8 port_num) { - return call_int_hook(ib_endport_manage_subnet, 0, sec, - dev_name, port_num); + return call_int_hook(ib_endport_manage_subnet, sec, dev_name, port_num); } EXPORT_SYMBOL(security_ib_endport_manage_subnet); @@ -5058,7 +5020,7 @@ EXPORT_SYMBOL(security_ib_endport_manage_subnet); */ int security_ib_alloc_security(void **sec) { - return call_int_hook(ib_alloc_security, 0, sec); + return call_int_hook(ib_alloc_security, sec); } EXPORT_SYMBOL(security_ib_alloc_security); @@ -5091,7 +5053,7 @@ int security_xfrm_policy_alloc(struct xfrm_sec_ctx **ctxp, struct xfrm_user_sec_ctx *sec_ctx, gfp_t gfp) { - return call_int_hook(xfrm_policy_alloc_security, 0, ctxp, sec_ctx, gfp); + return call_int_hook(xfrm_policy_alloc_security, ctxp, sec_ctx, gfp); } EXPORT_SYMBOL(security_xfrm_policy_alloc); @@ -5108,7 +5070,7 @@ EXPORT_SYMBOL(security_xfrm_policy_alloc); int security_xfrm_policy_clone(struct xfrm_sec_ctx *old_ctx, struct xfrm_sec_ctx **new_ctxp) { - return call_int_hook(xfrm_policy_clone_security, 0, old_ctx, new_ctxp); + return call_int_hook(xfrm_policy_clone_security, old_ctx, new_ctxp); } /** @@ -5133,7 +5095,7 @@ EXPORT_SYMBOL(security_xfrm_policy_free); */ int security_xfrm_policy_delete(struct xfrm_sec_ctx *ctx) { - return call_int_hook(xfrm_policy_delete_security, 0, ctx); + return call_int_hook(xfrm_policy_delete_security, ctx); } /** @@ -5150,7 +5112,7 @@ int security_xfrm_policy_delete(struct xfrm_sec_ctx *ctx) int security_xfrm_state_alloc(struct xfrm_state *x, struct xfrm_user_sec_ctx *sec_ctx) { - return call_int_hook(xfrm_state_alloc, 0, x, sec_ctx); + return call_int_hook(xfrm_state_alloc, x, sec_ctx); } EXPORT_SYMBOL(security_xfrm_state_alloc); @@ -5169,7 +5131,7 @@ EXPORT_SYMBOL(security_xfrm_state_alloc); int security_xfrm_state_alloc_acquire(struct xfrm_state *x, struct xfrm_sec_ctx *polsec, u32 secid) { - return call_int_hook(xfrm_state_alloc_acquire, 0, x, polsec, secid); + return call_int_hook(xfrm_state_alloc_acquire, x, polsec, secid); } /** @@ -5182,7 +5144,7 @@ int security_xfrm_state_alloc_acquire(struct xfrm_state *x, */ int security_xfrm_state_delete(struct xfrm_state *x) { - return call_int_hook(xfrm_state_delete_security, 0, x); + return call_int_hook(xfrm_state_delete_security, x); } EXPORT_SYMBOL(security_xfrm_state_delete); @@ -5211,7 +5173,7 @@ void security_xfrm_state_free(struct xfrm_state *x) */ int security_xfrm_policy_lookup(struct xfrm_sec_ctx *ctx, u32 fl_secid) { - return call_int_hook(xfrm_policy_lookup, 0, ctx, fl_secid); + return call_int_hook(xfrm_policy_lookup, ctx, fl_secid); } /** @@ -5259,12 +5221,12 @@ int security_xfrm_state_pol_flow_match(struct xfrm_state *x, */ int security_xfrm_decode_session(struct sk_buff *skb, u32 *secid) { - return call_int_hook(xfrm_decode_session, 0, skb, secid, 1); + return call_int_hook(xfrm_decode_session, skb, secid, 1); } void security_skb_classify_flow(struct sk_buff *skb, struct flowi_common *flic) { - int rc = call_int_hook(xfrm_decode_session, 0, skb, &flic->flowic_secid, + int rc = call_int_hook(xfrm_decode_session, skb, &flic->flowic_secid, 0); BUG_ON(rc); @@ -5287,7 +5249,7 @@ EXPORT_SYMBOL(security_skb_classify_flow); int security_key_alloc(struct key *key, const struct cred *cred, unsigned long flags) { - return call_int_hook(key_alloc, 0, key, cred, flags); + return call_int_hook(key_alloc, key, cred, flags); } /** @@ -5314,7 +5276,7 @@ void security_key_free(struct key *key) int security_key_permission(key_ref_t key_ref, const struct cred *cred, enum key_need_perm need_perm) { - return call_int_hook(key_permission, 0, key_ref, cred, need_perm); + return call_int_hook(key_permission, key_ref, cred, need_perm); } /** @@ -5333,7 +5295,26 @@ int security_key_permission(key_ref_t key_ref, const struct cred *cred, int security_key_getsecurity(struct key *key, char **buffer) { *buffer = NULL; - return call_int_hook(key_getsecurity, 0, key, buffer); + return call_int_hook(key_getsecurity, key, buffer); +} + +/** + * security_key_post_create_or_update() - Notification of key create or update + * @keyring: keyring to which the key is linked to + * @key: created or updated key + * @payload: data used to instantiate or update the key + * @payload_len: length of payload + * @flags: key flags + * @create: flag indicating whether the key was created or updated + * + * Notify the caller of a key creation or update. + */ +void security_key_post_create_or_update(struct key *keyring, struct key *key, + const void *payload, size_t payload_len, + unsigned long flags, bool create) +{ + call_void_hook(key_post_create_or_update, keyring, key, payload, + payload_len, flags, create); } #endif /* CONFIG_KEYS */ @@ -5352,7 +5333,7 @@ int security_key_getsecurity(struct key *key, char **buffer) */ int security_audit_rule_init(u32 field, u32 op, char *rulestr, void **lsmrule) { - return call_int_hook(audit_rule_init, 0, field, op, rulestr, lsmrule); + return call_int_hook(audit_rule_init, field, op, rulestr, lsmrule); } /** @@ -5366,7 +5347,7 @@ int security_audit_rule_init(u32 field, u32 op, char *rulestr, void **lsmrule) */ int security_audit_rule_known(struct audit_krule *krule) { - return call_int_hook(audit_rule_known, 0, krule); + return call_int_hook(audit_rule_known, krule); } /** @@ -5396,7 +5377,7 @@ void security_audit_rule_free(void *lsmrule) */ int security_audit_rule_match(u32 secid, u32 field, u32 op, void *lsmrule) { - return call_int_hook(audit_rule_match, 0, secid, field, op, lsmrule); + return call_int_hook(audit_rule_match, secid, field, op, lsmrule); } #endif /* CONFIG_AUDIT */ @@ -5415,7 +5396,7 @@ int security_audit_rule_match(u32 secid, u32 field, u32 op, void *lsmrule) */ int security_bpf(int cmd, union bpf_attr *attr, unsigned int size) { - return call_int_hook(bpf, 0, cmd, attr, size); + return call_int_hook(bpf, cmd, attr, size); } /** @@ -5430,7 +5411,7 @@ int security_bpf(int cmd, union bpf_attr *attr, unsigned int size) */ int security_bpf_map(struct bpf_map *map, fmode_t fmode) { - return call_int_hook(bpf_map, 0, map, fmode); + return call_int_hook(bpf_map, map, fmode); } /** @@ -5444,7 +5425,7 @@ int security_bpf_map(struct bpf_map *map, fmode_t fmode) */ int security_bpf_prog(struct bpf_prog *prog) { - return call_int_hook(bpf_prog, 0, prog); + return call_int_hook(bpf_prog, prog); } /** @@ -5461,7 +5442,7 @@ int security_bpf_prog(struct bpf_prog *prog) int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr, struct bpf_token *token) { - return call_int_hook(bpf_map_create, 0, map, attr, token); + return call_int_hook(bpf_map_create, map, attr, token); } /** @@ -5479,7 +5460,7 @@ int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr, int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, struct bpf_token *token) { - return call_int_hook(bpf_prog_load, 0, prog, attr, token); + return call_int_hook(bpf_prog_load, prog, attr, token); } /** @@ -5496,7 +5477,7 @@ int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr, int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr, struct path *path) { - return call_int_hook(bpf_token_create, 0, token, attr, path); + return call_int_hook(bpf_token_create, token, attr, path); } /** @@ -5512,7 +5493,7 @@ int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr, */ int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cmd) { - return call_int_hook(bpf_token_cmd, 0, token, cmd); + return call_int_hook(bpf_token_cmd, token, cmd); } /** @@ -5528,7 +5509,7 @@ int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cmd) */ int security_bpf_token_capable(const struct bpf_token *token, int cap) { - return call_int_hook(bpf_token_capable, 0, token, cap); + return call_int_hook(bpf_token_capable, token, cap); } /** @@ -5576,7 +5557,7 @@ void security_bpf_token_free(struct bpf_token *token) */ int security_locked_down(enum lockdown_reason what) { - return call_int_hook(locked_down, 0, what); + return call_int_hook(locked_down, what); } EXPORT_SYMBOL(security_locked_down); @@ -5592,7 +5573,7 @@ EXPORT_SYMBOL(security_locked_down); */ int security_perf_event_open(struct perf_event_attr *attr, int type) { - return call_int_hook(perf_event_open, 0, attr, type); + return call_int_hook(perf_event_open, attr, type); } /** @@ -5605,7 +5586,7 @@ int security_perf_event_open(struct perf_event_attr *attr, int type) */ int security_perf_event_alloc(struct perf_event *event) { - return call_int_hook(perf_event_alloc, 0, event); + return call_int_hook(perf_event_alloc, event); } /** @@ -5629,7 +5610,7 @@ void security_perf_event_free(struct perf_event *event) */ int security_perf_event_read(struct perf_event *event) { - return call_int_hook(perf_event_read, 0, event); + return call_int_hook(perf_event_read, event); } /** @@ -5642,7 +5623,7 @@ int security_perf_event_read(struct perf_event *event) */ int security_perf_event_write(struct perf_event *event) { - return call_int_hook(perf_event_write, 0, event); + return call_int_hook(perf_event_write, event); } #endif /* CONFIG_PERF_EVENTS */ @@ -5658,7 +5639,7 @@ int security_perf_event_write(struct perf_event *event) */ int security_uring_override_creds(const struct cred *new) { - return call_int_hook(uring_override_creds, 0, new); + return call_int_hook(uring_override_creds, new); } /** @@ -5671,7 +5652,7 @@ int security_uring_override_creds(const struct cred *new) */ int security_uring_sqpoll(void) { - return call_int_hook(uring_sqpoll, 0); + return call_int_hook(uring_sqpoll); } /** @@ -5684,6 +5665,6 @@ int security_uring_sqpoll(void) */ int security_uring_cmd(struct io_uring_cmd *ioucmd) { - return call_int_hook(uring_cmd, 0, ioucmd); + return call_int_hook(uring_cmd, ioucmd); } #endif /* CONFIG_IO_URING */ diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 860e558e9fd5..8db4875164ab 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -3135,7 +3135,8 @@ static int selinux_inode_permission(struct inode *inode, int mask) return rc; } -static int selinux_inode_setattr(struct dentry *dentry, struct iattr *iattr) +static int selinux_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) { const struct cred *cred = current_cred(); struct inode *inode = d_backing_inode(dentry); diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index b18b50232947..28be26712396 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -1238,12 +1238,14 @@ static int smack_inode_permission(struct inode *inode, int mask) /** * smack_inode_setattr - Smack check for setting attributes + * @idmap: idmap of the mount * @dentry: the object * @iattr: for the force flag * * Returns 0 if access is permitted, an error code otherwise */ -static int smack_inode_setattr(struct dentry *dentry, struct iattr *iattr) +static int smack_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) { struct smk_audit_info ad; int rc; diff --git a/sound/hda/hdac_device.c b/sound/hda/hdac_device.c index 7f7b67fe1b65..068c16e52dff 100644 --- a/sound/hda/hdac_device.c +++ b/sound/hda/hdac_device.c @@ -612,7 +612,7 @@ EXPORT_SYMBOL_GPL(snd_hdac_power_up_pm); int snd_hdac_keep_power_up(struct hdac_device *codec) { if (!atomic_inc_not_zero(&codec->in_pm)) { - int ret = pm_runtime_get_if_active(&codec->dev, true); + int ret = pm_runtime_get_if_active(&codec->dev); if (!ret) return -1; if (ret < 0) diff --git a/tools/include/linux/slab.h b/tools/include/linux/slab.h index 311759ea25e9..51b25e9c4ec7 100644 --- a/tools/include/linux/slab.h +++ b/tools/include/linux/slab.h @@ -18,7 +18,6 @@ bool slab_is_available(void); enum slab_state { DOWN, PARTIAL, - PARTIAL_NODE, UP, FULL }; diff --git a/tools/power/cpupower/man/cpupower-frequency-info.1 b/tools/power/cpupower/man/cpupower-frequency-info.1 index dd545b499480..47fdd7218748 100644 --- a/tools/power/cpupower/man/cpupower-frequency-info.1 +++ b/tools/power/cpupower/man/cpupower-frequency-info.1 @@ -32,7 +32,7 @@ Gets the currently used cpufreq policy. \fB\-g\fR \fB\-\-governors\fR Determines available cpufreq governors. .TP -\fB\-a\fR \fB\-\-related\-cpus\fR +\fB\-r\fR \fB\-\-related\-cpus\fR Determines which CPUs run at the same hardware frequency. .TP \fB\-a\fR \fB\-\-affected\-cpus\fR diff --git a/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c b/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c index 5fd9e594079c..ebda9c366b2b 100644 --- a/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c +++ b/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c @@ -1241,6 +1241,7 @@ unsigned int get_pkg_num(int cpu) retval = fscanf(fp, "%d\n", &pkg); if (retval != 1) errx(1, "%s: failed to parse", pathname); + fclose(fp); return pkg; } diff --git a/tools/testing/selftests/gpio/gpio-mockup.sh b/tools/testing/selftests/gpio/gpio-mockup.sh index 0d6c5f7f95d2..fc2dd4c24d06 100755 --- a/tools/testing/selftests/gpio/gpio-mockup.sh +++ b/tools/testing/selftests/gpio/gpio-mockup.sh @@ -377,13 +377,10 @@ if [ "$full_test" ]; then insmod_test "0,32,32,44,-1,22,-1,31" 32 12 22 31 fi echo "2. Module load error tests" -echo "2.1 gpio overflow" -# Currently: The max number of gpio(1024) is defined in arm architecture. -insmod_test "-1,1024" +echo "2.1 no lines defined" +insmod_test "0,0" if [ "$full_test" ]; then - echo "2.2 no lines defined" - insmod_test "0,0" - echo "2.3 ignore range overlap" + echo "2.2 ignore range overlap" insmod_test "0,32,0,1" 32 insmod_test "0,32,1,5" 32 insmod_test "0,32,30,35" 32 diff --git a/tools/testing/selftests/lsm/lsm_list_modules_test.c b/tools/testing/selftests/lsm/lsm_list_modules_test.c index 9df29b1e3497..4d5d4cee2586 100644 --- a/tools/testing/selftests/lsm/lsm_list_modules_test.c +++ b/tools/testing/selftests/lsm/lsm_list_modules_test.c @@ -122,6 +122,12 @@ TEST(correct_lsm_list_modules) case LSM_ID_LANDLOCK: name = "landlock"; break; + case LSM_ID_IMA: + name = "ima"; + break; + case LSM_ID_EVM: + name = "evm"; + break; default: name = "INVALID"; break;