tmakin/ack-tegra
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge 7aefa59899 ("powerpc: assert_pte_locked() use pte_offset_map_ro_nolock()") into android-mainline

Browse Source 
Steps on the way to 6.13-rc1

Change-Id: I88a3006f8619054992a6d5b528a1bbb198562b2d
Signed-off-by: Greg Kroah-Hartman <gregkh@google.com>
This commit is contained in:
Greg Kroah-Hartman committed 2025-02-21 13:08:32 +01:00
2 parents 49f6404645 7aefa59899
commit 36b705aced
37 files changed
+624 -263
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Documentation/admin-guide/blockdev/zram.rst
+2
View File
@@ -47,6 +47,8 @@ The list of possible return codes:
-ENOMEM zram was not able to allocate enough memory to fulfil your
needs.
-EINVAL invalid input has been provided.
-EAGAIN re-try operation later (e.g. when attempting to run recompress
and writeback simultaneously).
======== =============================================================
If you use 'echo', the returned value is set by the 'echo' utility,
Documentation/mm/split_page_table_lock.rst
+7
View File
@@ -19,6 +19,13 @@ There are helpers to lock/unlock a table and other accessor functions:
- pte_offset_map_nolock()
maps PTE, returns pointer to PTE with pointer to its PTE table
lock (not taken), or returns NULL if no PTE table;
- pte_offset_map_ro_nolock()
maps PTE, returns pointer to PTE with pointer to its PTE table
lock (not taken), or returns NULL if no PTE table;
- pte_offset_map_rw_nolock()
maps PTE, returns pointer to PTE with pointer to its PTE table
lock (not taken) and the value of its pmd entry, or returns NULL
if no PTE table;
- pte_offset_map()
maps PTE, returns pointer to PTE, or returns NULL if no PTE table;
- pte_unmap()
arch/powerpc/mm/pgtable.c
+1 -1
View File
@@ -398,7 +398,7 @@ void assert_pte_locked(struct mm_struct *mm, unsigned long addr)
*/
if (pmd_none(*pmd))
return;
pte = pte_offset_map_nolock(mm, pmd, addr, &ptl);
pte = pte_offset_map_ro_nolock(mm, pmd, addr, &ptl);
BUG_ON(!pte);
assert_spin_locked(ptl);
pte_unmap(pte);
drivers/block/zram/zram_drv.c
+276 -94
View File
@@ -178,11 +178,105 @@ static inline u32 zram_get_priority(struct zram *zram, u32 index)
static void zram_accessed(struct zram *zram, u32 index)
{
zram_clear_flag(zram, index, ZRAM_IDLE);
zram_clear_flag(zram, index, ZRAM_PP_SLOT);
#ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME
zram->table[index].ac_time = ktime_get_boottime();
#endif
}
#if defined CONFIG_ZRAM_WRITEBACK || defined CONFIG_ZRAM_MULTI_COMP
struct zram_pp_slot {
unsigned long index;
struct list_head entry;
};
/*
* A post-processing bucket is, essentially, a size class, this defines
* the range (in bytes) of pp-slots sizes in particular bucket.
*/
#define PP_BUCKET_SIZE_RANGE 64
#define NUM_PP_BUCKETS ((PAGE_SIZE / PP_BUCKET_SIZE_RANGE) + 1)
struct zram_pp_ctl {
struct list_head pp_buckets[NUM_PP_BUCKETS];
};
static struct zram_pp_ctl *init_pp_ctl(void)
{
struct zram_pp_ctl *ctl;
u32 idx;
ctl = kmalloc(sizeof(*ctl), GFP_KERNEL);
if (!ctl)
return NULL;
for (idx = 0; idx < NUM_PP_BUCKETS; idx++)
INIT_LIST_HEAD(&ctl->pp_buckets[idx]);
return ctl;
}
static void release_pp_slot(struct zram *zram, struct zram_pp_slot *pps)
{
list_del_init(&pps->entry);
zram_slot_lock(zram, pps->index);
zram_clear_flag(zram, pps->index, ZRAM_PP_SLOT);
zram_slot_unlock(zram, pps->index);
kfree(pps);
}
static void release_pp_ctl(struct zram *zram, struct zram_pp_ctl *ctl)
{
u32 idx;
if (!ctl)
return;
for (idx = 0; idx < NUM_PP_BUCKETS; idx++) {
while (!list_empty(&ctl->pp_buckets[idx])) {
struct zram_pp_slot *pps;
pps = list_first_entry(&ctl->pp_buckets[idx],
struct zram_pp_slot,
entry);
release_pp_slot(zram, pps);
}
}
kfree(ctl);
}
static void place_pp_slot(struct zram *zram, struct zram_pp_ctl *ctl,
struct zram_pp_slot *pps)
{
u32 idx;
idx = zram_get_obj_size(zram, pps->index) / PP_BUCKET_SIZE_RANGE;
list_add(&pps->entry, &ctl->pp_buckets[idx]);
zram_set_flag(zram, pps->index, ZRAM_PP_SLOT);
}
static struct zram_pp_slot *select_pp_slot(struct zram_pp_ctl *ctl)
{
struct zram_pp_slot *pps = NULL;
s32 idx = NUM_PP_BUCKETS - 1;
/* The higher the bucket id the more optimal slot post-processing is */
while (idx >= 0) {
pps = list_first_entry_or_null(&ctl->pp_buckets[idx],
struct zram_pp_slot,
entry);
if (pps)
break;
idx--;
}
return pps;
}
#endif
static inline void update_used_max(struct zram *zram,
const unsigned long pages)
{
@@ -296,19 +390,26 @@ static void mark_idle(struct zram *zram, ktime_t cutoff)
for (index = 0; index < nr_pages; index++) {
/*
* Do not mark ZRAM_UNDER_WB slot as ZRAM_IDLE to close race.
* See the comment in writeback_store.
* Do not mark ZRAM_SAME slots as ZRAM_IDLE, because no
* post-processing (recompress, writeback) happens to the
* ZRAM_SAME slot.
*
* And ZRAM_WB slots simply cannot be ZRAM_IDLE.
*/
zram_slot_lock(zram, index);
if (zram_allocated(zram, index) &&
!zram_test_flag(zram, index, ZRAM_UNDER_WB)) {
#ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME
is_idle = !cutoff || ktime_after(cutoff,
zram->table[index].ac_time);
#endif
if (is_idle)
zram_set_flag(zram, index, ZRAM_IDLE);
if (!zram_allocated(zram, index) ||
zram_test_flag(zram, index, ZRAM_WB) ||
zram_test_flag(zram, index, ZRAM_SAME)) {
zram_slot_unlock(zram, index);
continue;
}
#ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME
is_idle = !cutoff ||
ktime_after(cutoff, zram->table[index].ac_time);
#endif
if (is_idle)
zram_set_flag(zram, index, ZRAM_IDLE);
zram_slot_unlock(zram, index);
}
}
@@ -587,11 +688,57 @@ static void read_from_bdev_async(struct zram *zram, struct page *page,
#define IDLE_WRITEBACK (1<<1)
#define INCOMPRESSIBLE_WRITEBACK (1<<2)
static int scan_slots_for_writeback(struct zram *zram, u32 mode,
unsigned long nr_pages,
unsigned long index,
struct zram_pp_ctl *ctl)
{
struct zram_pp_slot *pps = NULL;
for (; nr_pages != 0; index++, nr_pages--) {
if (!pps)
pps = kmalloc(sizeof(*pps), GFP_KERNEL);
if (!pps)
return -ENOMEM;
INIT_LIST_HEAD(&pps->entry);
zram_slot_lock(zram, index);
if (!zram_allocated(zram, index))
goto next;
if (zram_test_flag(zram, index, ZRAM_WB) ||
zram_test_flag(zram, index, ZRAM_SAME))
goto next;
if (mode & IDLE_WRITEBACK &&
!zram_test_flag(zram, index, ZRAM_IDLE))
goto next;
if (mode & HUGE_WRITEBACK &&
!zram_test_flag(zram, index, ZRAM_HUGE))
goto next;
if (mode & INCOMPRESSIBLE_WRITEBACK &&
!zram_test_flag(zram, index, ZRAM_INCOMPRESSIBLE))
goto next;
pps->index = index;
place_pp_slot(zram, ctl, pps);
pps = NULL;
next:
zram_slot_unlock(zram, index);
}
kfree(pps);
return 0;
}
static ssize_t writeback_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t len)
{
struct zram *zram = dev_to_zram(dev);
unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
struct zram_pp_ctl *ctl = NULL;
struct zram_pp_slot *pps;
unsigned long index = 0;
struct bio bio;
struct bio_vec bio_vec;
@@ -626,6 +773,12 @@ static ssize_t writeback_store(struct device *dev,
goto release_init_lock;
}
/* Do not permit concurrent post-processing actions. */
if (atomic_xchg(&zram->pp_in_progress, 1)) {
up_read(&zram->init_lock);
return -EAGAIN;
}
if (!zram->backing_dev) {
ret = -ENODEV;
goto release_init_lock;
@@ -637,7 +790,15 @@ static ssize_t writeback_store(struct device *dev,
goto release_init_lock;
}
for (; nr_pages != 0; index++, nr_pages--) {
ctl = init_pp_ctl();
if (!ctl) {
ret = -ENOMEM;
goto release_init_lock;
}
scan_slots_for_writeback(zram, mode, nr_pages, index, ctl);
while ((pps = select_pp_slot(ctl))) {
spin_lock(&zram->wb_limit_lock);
if (zram->wb_limit_enable && !zram->bd_wb_limit) {
spin_unlock(&zram->wb_limit_lock);
@@ -654,38 +815,20 @@ static ssize_t writeback_store(struct device *dev,
}
}
index = pps->index;
zram_slot_lock(zram, index);
if (!zram_allocated(zram, index))
goto next;
if (zram_test_flag(zram, index, ZRAM_WB) ||
zram_test_flag(zram, index, ZRAM_SAME) ||
zram_test_flag(zram, index, ZRAM_UNDER_WB))
goto next;
if (mode & IDLE_WRITEBACK &&
!zram_test_flag(zram, index, ZRAM_IDLE))
goto next;
if (mode & HUGE_WRITEBACK &&
!zram_test_flag(zram, index, ZRAM_HUGE))
goto next;
if (mode & INCOMPRESSIBLE_WRITEBACK &&
!zram_test_flag(zram, index, ZRAM_INCOMPRESSIBLE))
goto next;
/*
* Clearing ZRAM_UNDER_WB is duty of caller.
* IOW, zram_free_page never clear it.
* scan_slots() sets ZRAM_PP_SLOT and relases slot lock, so
* slots can change in the meantime. If slots are accessed or
* freed they lose ZRAM_PP_SLOT flag and hence we don't
* post-process them.
*/
zram_set_flag(zram, index, ZRAM_UNDER_WB);
/* Need for hugepage writeback racing */
zram_set_flag(zram, index, ZRAM_IDLE);
if (!zram_test_flag(zram, index, ZRAM_PP_SLOT))
goto next;
zram_slot_unlock(zram, index);
if (zram_read_page(zram, page, index, NULL)) {
zram_slot_lock(zram, index);
zram_clear_flag(zram, index, ZRAM_UNDER_WB);
zram_clear_flag(zram, index, ZRAM_IDLE);
zram_slot_unlock(zram, index);
release_pp_slot(zram, pps);
continue;
}
@@ -700,10 +843,7 @@ static ssize_t writeback_store(struct device *dev,
*/
err = submit_bio_wait(&bio);
if (err) {
zram_slot_lock(zram, index);
zram_clear_flag(zram, index, ZRAM_UNDER_WB);
zram_clear_flag(zram, index, ZRAM_IDLE);
zram_slot_unlock(zram, index);
release_pp_slot(zram, pps);
/*
* BIO errors are not fatal, we continue and simply
* attempt to writeback the remaining objects (pages).
@@ -717,25 +857,19 @@ static ssize_t writeback_store(struct device *dev,
}
atomic64_inc(&zram->stats.bd_writes);
/*
* We released zram_slot_lock so need to check if the slot was
* changed. If there is freeing for the slot, we can catch it
* easily by zram_allocated.
* A subtle case is the slot is freed/reallocated/marked as
* ZRAM_IDLE again. To close the race, idle_store doesn't
* mark ZRAM_IDLE once it found the slot was ZRAM_UNDER_WB.
* Thus, we could close the race by checking ZRAM_IDLE bit.
*/
zram_slot_lock(zram, index);
if (!zram_allocated(zram, index) ||
!zram_test_flag(zram, index, ZRAM_IDLE)) {
zram_clear_flag(zram, index, ZRAM_UNDER_WB);
zram_clear_flag(zram, index, ZRAM_IDLE);
/*
* Same as above, we release slot lock during writeback so
* slot can change under us: slot_free() or slot_free() and
* reallocation (zram_write_page()). In both cases slot loses
* ZRAM_PP_SLOT flag. No concurrent post-processing can set
* ZRAM_PP_SLOT on such slots until current post-processing
* finishes.
*/
if (!zram_test_flag(zram, index, ZRAM_PP_SLOT))
goto next;
}
zram_free_page(zram, index);
zram_clear_flag(zram, index, ZRAM_UNDER_WB);
zram_set_flag(zram, index, ZRAM_WB);
zram_set_element(zram, index, blk_idx);
blk_idx = 0;
@@ -746,12 +880,15 @@ static ssize_t writeback_store(struct device *dev,
spin_unlock(&zram->wb_limit_lock);
next:
zram_slot_unlock(zram, index);
release_pp_slot(zram, pps);
}
if (blk_idx)
free_block_bdev(zram, blk_idx);
__free_page(page);
release_init_lock:
release_pp_ctl(zram, ctl);
atomic_set(&zram->pp_in_progress, 0);
up_read(&zram->init_lock);
return ret;
@@ -1342,19 +1479,17 @@ static void zram_free_page(struct zram *zram, size_t index)
#ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME
zram->table[index].ac_time = 0;
#endif
if (zram_test_flag(zram, index, ZRAM_IDLE))
zram_clear_flag(zram, index, ZRAM_IDLE);
zram_clear_flag(zram, index, ZRAM_IDLE);
zram_clear_flag(zram, index, ZRAM_INCOMPRESSIBLE);
zram_clear_flag(zram, index, ZRAM_PP_SLOT);
zram_set_priority(zram, index, 0);
if (zram_test_flag(zram, index, ZRAM_HUGE)) {
zram_clear_flag(zram, index, ZRAM_HUGE);
atomic64_dec(&zram->stats.huge_pages);
}
if (zram_test_flag(zram, index, ZRAM_INCOMPRESSIBLE))
zram_clear_flag(zram, index, ZRAM_INCOMPRESSIBLE);
zram_set_priority(zram, index, 0);
if (zram_test_flag(zram, index, ZRAM_WB)) {
zram_clear_flag(zram, index, ZRAM_WB);
free_block_bdev(zram, zram_get_element(zram, index));
@@ -1378,13 +1513,11 @@ static void zram_free_page(struct zram *zram, size_t index)
zs_free(zram->mem_pool, handle);
atomic64_sub(zram_get_obj_size(zram, index),
&zram->stats.compr_data_size);
&zram->stats.compr_data_size);
out:
atomic64_dec(&zram->stats.pages_stored);
zram_set_handle(zram, index, 0);
zram_set_obj_size(zram, index, 0);
WARN_ON_ONCE(zram->table[index].flags &
~(1UL << ZRAM_UNDER_WB));
}
/*
@@ -1648,6 +1781,52 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec,
}
#ifdef CONFIG_ZRAM_MULTI_COMP
#define RECOMPRESS_IDLE (1 << 0)
#define RECOMPRESS_HUGE (1 << 1)
static int scan_slots_for_recompress(struct zram *zram, u32 mode,
struct zram_pp_ctl *ctl)
{
unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
struct zram_pp_slot *pps = NULL;
unsigned long index;
for (index = 0; index < nr_pages; index++) {
if (!pps)
pps = kmalloc(sizeof(*pps), GFP_KERNEL);
if (!pps)
return -ENOMEM;
INIT_LIST_HEAD(&pps->entry);
zram_slot_lock(zram, index);
if (!zram_allocated(zram, index))
goto next;
if (mode & RECOMPRESS_IDLE &&
!zram_test_flag(zram, index, ZRAM_IDLE))
goto next;
if (mode & RECOMPRESS_HUGE &&
!zram_test_flag(zram, index, ZRAM_HUGE))
goto next;
if (zram_test_flag(zram, index, ZRAM_WB) ||
zram_test_flag(zram, index, ZRAM_SAME) ||
zram_test_flag(zram, index, ZRAM_INCOMPRESSIBLE))
goto next;
pps->index = index;
place_pp_slot(zram, ctl, pps);
pps = NULL;
next:
zram_slot_unlock(zram, index);
}
kfree(pps);
return 0;
}
/*
* This function will decompress (unless it's ZRAM_HUGE) the page and then
* attempt to compress it using provided compression algorithm priority
@@ -1655,7 +1834,7 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec,
*
* Corresponding ZRAM slot should be locked.
*/
static int zram_recompress(struct zram *zram, u32 index, struct page *page,
static int recompress_slot(struct zram *zram, u32 index, struct page *page,
u64 *num_recomp_pages, u32 threshold, u32 prio,
u32 prio_max)
{
@@ -1798,20 +1977,17 @@ static int zram_recompress(struct zram *zram, u32 index, struct page *page,
return 0;
}
#define RECOMPRESS_IDLE (1 << 0)
#define RECOMPRESS_HUGE (1 << 1)
static ssize_t recompress_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t len)
{
u32 prio = ZRAM_SECONDARY_COMP, prio_max = ZRAM_MAX_COMPS;
struct zram *zram = dev_to_zram(dev);
unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
char *args, *param, *val, *algo = NULL;
u64 num_recomp_pages = ULLONG_MAX;
struct zram_pp_ctl *ctl = NULL;
struct zram_pp_slot *pps;
u32 mode = 0, threshold = 0;
unsigned long index;
struct page *page;
ssize_t ret;
@@ -1881,6 +2057,12 @@ static ssize_t recompress_store(struct device *dev,
goto release_init_lock;
}
/* Do not permit concurrent post-processing actions. */
if (atomic_xchg(&zram->pp_in_progress, 1)) {
up_read(&zram->init_lock);
return -EAGAIN;
}
if (algo) {
bool found = false;
@@ -1907,36 +2089,32 @@ static ssize_t recompress_store(struct device *dev,
goto release_init_lock;
}
ctl = init_pp_ctl();
if (!ctl) {
ret = -ENOMEM;
goto release_init_lock;
}
scan_slots_for_recompress(zram, mode, ctl);
ret = len;
for (index = 0; index < nr_pages; index++) {
while ((pps = select_pp_slot(ctl))) {
int err = 0;
if (!num_recomp_pages)
break;
zram_slot_lock(zram, index);
if (!zram_allocated(zram, index))
zram_slot_lock(zram, pps->index);
if (!zram_test_flag(zram, pps->index, ZRAM_PP_SLOT))
goto next;
if (mode & RECOMPRESS_IDLE &&
!zram_test_flag(zram, index, ZRAM_IDLE))
goto next;
if (mode & RECOMPRESS_HUGE &&
!zram_test_flag(zram, index, ZRAM_HUGE))
goto next;
if (zram_test_flag(zram, index, ZRAM_WB) ||
zram_test_flag(zram, index, ZRAM_UNDER_WB) ||
zram_test_flag(zram, index, ZRAM_SAME) ||
zram_test_flag(zram, index, ZRAM_INCOMPRESSIBLE))
goto next;
err = zram_recompress(zram, index, page, &num_recomp_pages,
threshold, prio, prio_max);
err = recompress_slot(zram, pps->index, page,
&num_recomp_pages, threshold,
prio, prio_max);
next:
zram_slot_unlock(zram, index);
zram_slot_unlock(zram, pps->index);
release_pp_slot(zram, pps);
if (err) {
ret = err;
break;
@@ -1948,6 +2126,8 @@ next:
__free_page(page);
release_init_lock:
release_pp_ctl(zram, ctl);
atomic_set(&zram->pp_in_progress, 0);
up_read(&zram->init_lock);
return ret;
}
@@ -2144,6 +2324,7 @@ static void zram_reset_device(struct zram *zram)
zram->disksize = 0;
zram_destroy_comps(zram);
memset(&zram->stats, 0, sizeof(zram->stats));
atomic_set(&zram->pp_in_progress, 0);
reset_bdev(zram);
comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor);
@@ -2381,6 +2562,7 @@ static int zram_add(void)
zram->disk->fops = &zram_devops;
zram->disk->private_data = zram;
snprintf(zram->disk->disk_name, 16, "zram%d", device_id);
atomic_set(&zram->pp_in_progress, 0);
/* Actual capacity set using sysfs (/sys/block/zram<id>/disksize */
set_capacity(zram->disk, 0);
drivers/block/zram/zram_drv.h
+2 -1
View File
@@ -47,7 +47,7 @@
enum zram_pageflags {
ZRAM_SAME = ZRAM_FLAG_SHIFT, /* Page consists the same element */
ZRAM_WB, /* page is stored on backing_device */
ZRAM_UNDER_WB, /* page is under writeback */
ZRAM_PP_SLOT, /* Selected for post-processing */
ZRAM_HUGE, /* Incompressible page */
ZRAM_IDLE, /* not accessed page since last idle marking */
ZRAM_INCOMPRESSIBLE, /* none of the algorithms could compress it */
@@ -139,5 +139,6 @@ struct zram {
#ifdef CONFIG_ZRAM_MEMORY_TRACKING
struct dentry *debugfs_dir;
#endif
atomic_t pp_in_progress;
};
#endif
fs/proc/base.c
+14 -12
View File
@@ -832,19 +832,21 @@ static const struct file_operations proc_single_file_operations = {
struct mm_struct *proc_mem_open(struct inode *inode, unsigned int mode)
{
struct task_struct *task = get_proc_task(inode);
struct mm_struct *mm = ERR_PTR(-ESRCH);
struct mm_struct *mm;
if (task) {
mm = mm_access(task, mode | PTRACE_MODE_FSCREDS);
put_task_struct(task);
if (!task)
return ERR_PTR(-ESRCH);
if (!IS_ERR_OR_NULL(mm)) {
/* ensure this mm_struct can't be freed */
mmgrab(mm);
/* but do not pin its memory */
mmput(mm);
}
}
mm = mm_access(task, mode | PTRACE_MODE_FSCREDS);
put_task_struct(task);
if (IS_ERR(mm))
return mm == ERR_PTR(-ESRCH) ? NULL : mm;
/* ensure this mm_struct can't be freed */
mmgrab(mm);
/* but do not pin its memory */
mmput(mm);
return mm;
}
@@ -2208,7 +2210,7 @@ static int map_files_d_revalidate(struct dentry *dentry, unsigned int flags)
goto out_notask;
mm = mm_access(task, PTRACE_MODE_READ_FSCREDS);
if (IS_ERR_OR_NULL(mm))
if (IS_ERR(mm))
goto out;
if (!dname_to_vma_addr(dentry, &vm_start, &vm_end)) {
include/linux/huge_mm.h
-1
View File
@@ -2,7 +2,6 @@
#ifndef _LINUX_HUGE_MM_H
#define _LINUX_HUGE_MM_H
#include <linux/sched/coredump.h>
#include <linux/mm_types.h>
#include <linux/fs.h> /* only for vma_is_dax() */
include/linux/kasan.h
+9 -3
View File
@@ -29,6 +29,9 @@ typedef unsigned int __bitwise kasan_vmalloc_flags_t;
#define KASAN_VMALLOC_VM_ALLOC ((__force kasan_vmalloc_flags_t)0x02u)
#define KASAN_VMALLOC_PROT_NORMAL ((__force kasan_vmalloc_flags_t)0x04u)
#define KASAN_VMALLOC_PAGE_RANGE 0x1 /* Apply exsiting page range */
#define KASAN_VMALLOC_TLB_FLUSH 0x2 /* TLB flush */
#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
#include <linux/pgtable.h>
@@ -564,7 +567,8 @@ void kasan_populate_early_vm_area_shadow(void *start, unsigned long size);
int kasan_populate_vmalloc(unsigned long addr, unsigned long size);
void kasan_release_vmalloc(unsigned long start, unsigned long end,
unsigned long free_region_start,
unsigned long free_region_end);
unsigned long free_region_end,
unsigned long flags);
#else /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
@@ -579,7 +583,8 @@ static inline int kasan_populate_vmalloc(unsigned long start,
static inline void kasan_release_vmalloc(unsigned long start,
unsigned long end,
unsigned long free_region_start,
unsigned long free_region_end) { }
unsigned long free_region_end,
unsigned long flags) { }
#endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
@@ -614,7 +619,8 @@ static inline int kasan_populate_vmalloc(unsigned long start,
static inline void kasan_release_vmalloc(unsigned long start,
unsigned long end,
unsigned long free_region_start,
unsigned long free_region_end) { }
unsigned long free_region_end,
unsigned long flags) { }
static inline void *kasan_unpoison_vmalloc(const void *start,
unsigned long size,
include/linux/khugepaged.h
-2
View File
@@ -2,8 +2,6 @@
#ifndef _LINUX_KHUGEPAGED_H
#define _LINUX_KHUGEPAGED_H
#include <linux/sched/coredump.h> /* MMF_VM_HUGEPAGE */
extern unsigned int khugepaged_max_ptes_none __read_mostly;
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
extern struct attribute_group khugepaged_attr_group;
include/linux/ksm.h
-1
View File
@@ -13,7 +13,6 @@
#include <linux/pagemap.h>
#include <linux/rmap.h>
#include <linux/sched.h>
#include <linux/sched/coredump.h>
#ifdef CONFIG_KSM
int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
include/linux/mempolicy.h
+1 -1
View File
@@ -47,7 +47,7 @@ struct mempolicy {
atomic_t refcnt;
unsigned short mode; /* See MPOL_* above */
unsigned short flags; /* See set_mempolicy() MPOL_F_* above */
nodemask_t nodes; /* interleave/bind/perfer */
nodemask_t nodes; /* interleave/bind/preferred/etc */
int home_node; /* Home node to use for MPOL_BIND and MPOL_PREFERRED_MANY */
union {
include/linux/mm.h
+5
View File
@@ -3030,6 +3030,11 @@ static inline pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
pte_t *pte_offset_map_nolock(struct mm_struct *mm, pmd_t *pmd,
unsigned long addr, spinlock_t **ptlp);
pte_t *pte_offset_map_ro_nolock(struct mm_struct *mm, pmd_t *pmd,
unsigned long addr, spinlock_t **ptlp);
pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd,
unsigned long addr, pmd_t *pmdvalp,
spinlock_t **ptlp);
#define pte_unmap_unlock(pte, ptl) do { \
spin_unlock(ptl); \
include/linux/mm_types.h
+84
View File
@@ -1535,4 +1535,88 @@ enum {
/* See also internal only FOLL flags in mm/internal.h */
};
/* mm flags */
/*
* The first two bits represent core dump modes for set-user-ID,
* the modes are SUID_DUMP_* defined in linux/sched/coredump.h
*/
#define MMF_DUMPABLE_BITS 2
#define MMF_DUMPABLE_MASK ((1 << MMF_DUMPABLE_BITS) - 1)
/* coredump filter bits */
#define MMF_DUMP_ANON_PRIVATE 2
#define MMF_DUMP_ANON_SHARED 3
#define MMF_DUMP_MAPPED_PRIVATE 4
#define MMF_DUMP_MAPPED_SHARED 5
#define MMF_DUMP_ELF_HEADERS 6
#define MMF_DUMP_HUGETLB_PRIVATE 7
#define MMF_DUMP_HUGETLB_SHARED 8
#define MMF_DUMP_DAX_PRIVATE 9
#define MMF_DUMP_DAX_SHARED 10
#define MMF_DUMP_FILTER_SHIFT MMF_DUMPABLE_BITS
#define MMF_DUMP_FILTER_BITS 9
#define MMF_DUMP_FILTER_MASK \
(((1 << MMF_DUMP_FILTER_BITS) - 1) << MMF_DUMP_FILTER_SHIFT)
#define MMF_DUMP_FILTER_DEFAULT \
((1 << MMF_DUMP_ANON_PRIVATE) | (1 << MMF_DUMP_ANON_SHARED) |\
(1 << MMF_DUMP_HUGETLB_PRIVATE) | MMF_DUMP_MASK_DEFAULT_ELF)
#ifdef CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS
# define MMF_DUMP_MASK_DEFAULT_ELF (1 << MMF_DUMP_ELF_HEADERS)
#else
# define MMF_DUMP_MASK_DEFAULT_ELF 0
#endif
/* leave room for more dump flags */
#define MMF_VM_MERGEABLE 16 /* KSM may merge identical pages */
#define MMF_VM_HUGEPAGE 17 /* set when mm is available for khugepaged */
/*
* This one-shot flag is dropped due to necessity of changing exe once again
* on NFS restore
*/
//#define MMF_EXE_FILE_CHANGED 18 /* see prctl_set_mm_exe_file() */
#define MMF_HAS_UPROBES 19 /* has uprobes */
#define MMF_RECALC_UPROBES 20 /* MMF_HAS_UPROBES can be wrong */
#define MMF_OOM_SKIP 21 /* mm is of no interest for the OOM killer */
#define MMF_UNSTABLE 22 /* mm is unstable for copy_from_user */
#define MMF_HUGE_ZERO_PAGE 23 /* mm has ever used the global huge zero page */
#define MMF_DISABLE_THP 24 /* disable THP for all VMAs */
#define MMF_DISABLE_THP_MASK (1 << MMF_DISABLE_THP)
#define MMF_OOM_REAP_QUEUED 25 /* mm was queued for oom_reaper */
#define MMF_MULTIPROCESS 26 /* mm is shared between processes */
/*
* MMF_HAS_PINNED: Whether this mm has pinned any pages. This can be either
* replaced in the future by mm.pinned_vm when it becomes stable, or grow into
* a counter on its own. We're aggresive on this bit for now: even if the
* pinned pages were unpinned later on, we'll still keep this bit set for the
* lifecycle of this mm, just for simplicity.
*/
#define MMF_HAS_PINNED 27 /* FOLL_PIN has run, never cleared */
#define MMF_HAS_MDWE 28
#define MMF_HAS_MDWE_MASK (1 << MMF_HAS_MDWE)
#define MMF_HAS_MDWE_NO_INHERIT 29
#define MMF_VM_MERGE_ANY 30
#define MMF_VM_MERGE_ANY_MASK (1 << MMF_VM_MERGE_ANY)
#define MMF_TOPDOWN 31 /* mm searches top down by default */
#define MMF_TOPDOWN_MASK (1 << MMF_TOPDOWN)
#define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK |\
MMF_DISABLE_THP_MASK | MMF_HAS_MDWE_MASK |\
MMF_VM_MERGE_ANY_MASK | MMF_TOPDOWN_MASK)
static inline unsigned long mmf_init_flags(unsigned long flags)
{
if (flags & (1UL << MMF_HAS_MDWE_NO_INHERIT))
flags &= ~((1UL << MMF_HAS_MDWE) |
(1UL << MMF_HAS_MDWE_NO_INHERIT));
return flags & MMF_INIT_MASK;
}
#endif /* _LINUX_MM_TYPES_H */
include/linux/oom.h
-1
View File
@@ -7,7 +7,6 @@
#include <linux/types.h>
#include <linux/nodemask.h>
#include <uapi/linux/oom.h>
#include <linux/sched/coredump.h> /* MMF_* */
#include <linux/mm.h> /* VM_FAULT* */
struct zonelist;
include/linux/sched/coredump.h
-82
View File
@@ -8,12 +8,6 @@
#define SUID_DUMP_USER 1 /* Dump as user of process */
#define SUID_DUMP_ROOT 2 /* Dump as root */
/* mm flags */
/* for SUID_DUMP_* above */
#define MMF_DUMPABLE_BITS 2
#define MMF_DUMPABLE_MASK ((1 << MMF_DUMPABLE_BITS) - 1)
extern void set_dumpable(struct mm_struct *mm, int value);
/*
* This returns the actual value of the suid_dumpable flag. For things
@@ -31,80 +25,4 @@ static inline int get_dumpable(struct mm_struct *mm)
return __get_dumpable(mm->flags);
}
/* coredump filter bits */
#define MMF_DUMP_ANON_PRIVATE 2
#define MMF_DUMP_ANON_SHARED 3
#define MMF_DUMP_MAPPED_PRIVATE 4
#define MMF_DUMP_MAPPED_SHARED 5
#define MMF_DUMP_ELF_HEADERS 6
#define MMF_DUMP_HUGETLB_PRIVATE 7
#define MMF_DUMP_HUGETLB_SHARED 8
#define MMF_DUMP_DAX_PRIVATE 9
#define MMF_DUMP_DAX_SHARED 10
#define MMF_DUMP_FILTER_SHIFT MMF_DUMPABLE_BITS
#define MMF_DUMP_FILTER_BITS 9
#define MMF_DUMP_FILTER_MASK \
(((1 << MMF_DUMP_FILTER_BITS) - 1) << MMF_DUMP_FILTER_SHIFT)
#define MMF_DUMP_FILTER_DEFAULT \
((1 << MMF_DUMP_ANON_PRIVATE) | (1 << MMF_DUMP_ANON_SHARED) |\
(1 << MMF_DUMP_HUGETLB_PRIVATE) | MMF_DUMP_MASK_DEFAULT_ELF)
#ifdef CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS
# define MMF_DUMP_MASK_DEFAULT_ELF (1 << MMF_DUMP_ELF_HEADERS)
#else
# define MMF_DUMP_MASK_DEFAULT_ELF 0
#endif
/* leave room for more dump flags */
#define MMF_VM_MERGEABLE 16 /* KSM may merge identical pages */
#define MMF_VM_HUGEPAGE 17 /* set when mm is available for
khugepaged */
/*
* This one-shot flag is dropped due to necessity of changing exe once again
* on NFS restore
*/
//#define MMF_EXE_FILE_CHANGED 18 /* see prctl_set_mm_exe_file() */
#define MMF_HAS_UPROBES 19 /* has uprobes */
#define MMF_RECALC_UPROBES 20 /* MMF_HAS_UPROBES can be wrong */
#define MMF_OOM_SKIP 21 /* mm is of no interest for the OOM killer */
#define MMF_UNSTABLE 22 /* mm is unstable for copy_from_user */
#define MMF_HUGE_ZERO_PAGE 23 /* mm has ever used the global huge zero page */
#define MMF_DISABLE_THP 24 /* disable THP for all VMAs */
#define MMF_DISABLE_THP_MASK (1 << MMF_DISABLE_THP)
#define MMF_OOM_REAP_QUEUED 25 /* mm was queued for oom_reaper */
#define MMF_MULTIPROCESS 26 /* mm is shared between processes */
/*
* MMF_HAS_PINNED: Whether this mm has pinned any pages. This can be either
* replaced in the future by mm.pinned_vm when it becomes stable, or grow into
* a counter on its own. We're aggresive on this bit for now: even if the
* pinned pages were unpinned later on, we'll still keep this bit set for the
* lifecycle of this mm, just for simplicity.
*/
#define MMF_HAS_PINNED 27 /* FOLL_PIN has run, never cleared */
#define MMF_HAS_MDWE 28
#define MMF_HAS_MDWE_MASK (1 << MMF_HAS_MDWE)
#define MMF_HAS_MDWE_NO_INHERIT 29
#define MMF_VM_MERGE_ANY 30
#define MMF_VM_MERGE_ANY_MASK (1 << MMF_VM_MERGE_ANY)
#define MMF_TOPDOWN 31 /* mm searches top down by default */
#define MMF_TOPDOWN_MASK (1 << MMF_TOPDOWN)
#define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK |\
MMF_DISABLE_THP_MASK | MMF_HAS_MDWE_MASK |\
MMF_VM_MERGE_ANY_MASK | MMF_TOPDOWN_MASK)
static inline unsigned long mmf_init_flags(unsigned long flags)
{
if (flags & (1UL << MMF_HAS_MDWE_NO_INHERIT))
flags &= ~((1UL << MMF_HAS_MDWE) |
(1UL << MMF_HAS_MDWE_NO_INHERIT));
return flags & MMF_INIT_MASK;
}
#endif /* _LINUX_SCHED_COREDUMP_H */
include/linux/shmem_fs.h
+6
View File
@@ -114,6 +114,7 @@ int shmem_unuse(unsigned int type);
unsigned long shmem_allowable_huge_orders(struct inode *inode,
struct vm_area_struct *vma, pgoff_t index,
loff_t write_end, bool shmem_huge_force);
bool shmem_hpage_pmd_enabled(void);
#else
static inline unsigned long shmem_allowable_huge_orders(struct inode *inode,
struct vm_area_struct *vma, pgoff_t index,
@@ -121,6 +122,11 @@ static inline unsigned long shmem_allowable_huge_orders(struct inode *inode,
{
return 0;
}
static inline bool shmem_hpage_pmd_enabled(void)
{
return false;
}
#endif
#ifdef CONFIG_SHMEM
kernel/events/uprobes.c
-1
View File
@@ -15,7 +15,6 @@
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/sched/coredump.h>
#include <linux/export.h>
#include <linux/rmap.h> /* anon_vma_prepare */
#include <linux/mmu_notifier.h>
kernel/fork.c
+3 -3
View File
@@ -16,7 +16,6 @@
#include <linux/slab.h>
#include <linux/sched/autogroup.h>
#include <linux/sched/mm.h>
#include <linux/sched/coredump.h>
#include <linux/sched/user.h>
#include <linux/sched/numa_balancing.h>
#include <linux/sched/stat.h>
@@ -1578,8 +1577,9 @@ struct mm_struct *mm_access(struct task_struct *task, unsigned int mode)
return ERR_PTR(err);
mm = get_task_mm(task);
if (mm && mm != current->mm &&
!ptrace_may_access(task, mode)) {
if (!mm) {
mm = ERR_PTR(-ESRCH);
} else if (mm != current->mm && !ptrace_may_access(task, mode)) {
mmput(mm);
mm = ERR_PTR(-EACCES);
}
lib/maple_tree.c
+3 -8
View File
@@ -120,7 +120,6 @@ static const unsigned char mt_min_slots[] = {
#define MAPLE_BIG_NODE_GAPS (MAPLE_ARANGE64_SLOTS * 2 + 1)
struct maple_big_node {
struct maple_pnode *parent;
unsigned long pivot[MAPLE_BIG_NODE_SLOTS - 1];
union {
struct maple_enode *slot[MAPLE_BIG_NODE_SLOTS];
@@ -1943,14 +1942,13 @@ static inline void mas_mab_cp(struct ma_state *mas, unsigned char mas_start,
for (; i < piv_end; i++, j++) {
b_node->pivot[j] = pivots[i];
if (unlikely(!b_node->pivot[j]))
break;
goto complete;
if (unlikely(mas->max == b_node->pivot[j]))
goto complete;
}
if (likely(i <= mas_end))
b_node->pivot[j] = mas_safe_pivot(mas, pivots, i, mt);
b_node->pivot[j] = mas_safe_pivot(mas, pivots, i, mt);
complete:
b_node->b_end = ++j;
@@ -3159,10 +3157,7 @@ static inline void mast_fill_bnode(struct maple_subtree_state *mast,
bool cp = true;
unsigned char split;
memset(mast->bn->gap, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->gap));
memset(mast->bn->slot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->slot));
memset(mast->bn->pivot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->pivot));
mast->bn->b_end = 0;
memset(mast->bn, 0, sizeof(struct maple_big_node));
if (mte_is_root(mas->node)) {
cp = false;
mm/damon/tests/vaddr-kunit.h
+1 -1
View File
@@ -66,7 +66,7 @@ failed:
static void damon_test_three_regions_in_vmas(struct kunit *test)
{
static struct mm_struct mm;
struct damon_addr_range regions[3] = {0,};
struct damon_addr_range regions[3] = {0};
/* 10-20-25, 200-210-220, 300-305, 307-330 */
struct vm_area_struct vmas[] = {
(struct vm_area_struct) {.vm_start = 10, .vm_end = 20},
mm/huge_memory.c
-1
View File
@@ -8,7 +8,6 @@
#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/sched/coredump.h>
#include <linux/sched/numa_balancing.h>
#include <linux/highmem.h>
#include <linux/hugetlb.h>
mm/kasan/shadow.c
+10 -4
View File
@@ -489,7 +489,8 @@ static int kasan_depopulate_vmalloc_pte(pte_t *ptep, unsigned long addr,
*/
void kasan_release_vmalloc(unsigned long start, unsigned long end,
unsigned long free_region_start,
unsigned long free_region_end)
unsigned long free_region_end,
unsigned long flags)
{
void *shadow_start, *shadow_end;
unsigned long region_start, region_end;
@@ -522,12 +523,17 @@ void kasan_release_vmalloc(unsigned long start, unsigned long end,
__memset(shadow_start, KASAN_SHADOW_INIT, shadow_end - shadow_start);
return;
}
apply_to_existing_page_range(&init_mm,
if (flags & KASAN_VMALLOC_PAGE_RANGE)
apply_to_existing_page_range(&init_mm,
(unsigned long)shadow_start,
size, kasan_depopulate_vmalloc_pte,
NULL);
flush_tlb_kernel_range((unsigned long)shadow_start,
(unsigned long)shadow_end);
if (flags & KASAN_VMALLOC_TLB_FLUSH)
flush_tlb_kernel_range((unsigned long)shadow_start,
(unsigned long)shadow_end);
}
}
mm/khugepaged.c
+5 -2
View File
@@ -4,7 +4,6 @@
#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/sched/coredump.h>
#include <linux/mmu_notifier.h>
#include <linux/rmap.h>
#include <linux/swap.h>
@@ -416,9 +415,11 @@ static inline int hpage_collapse_test_exit_or_disable(struct mm_struct *mm)
static bool hugepage_pmd_enabled(void)
{
/*
* We cover both the anon and the file-backed case here; file-backed
* We cover the anon, shmem and the file-backed case here; file-backed
* hugepages, when configured in, are determined by the global control.
* Anon pmd-sized hugepages are determined by the pmd-size control.
* Shmem pmd-sized hugepages are also determined by its pmd-size control,
* except when the global shmem_huge is set to SHMEM_HUGE_DENY.
*/
if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) &&
hugepage_global_enabled())
@@ -430,6 +431,8 @@ static bool hugepage_pmd_enabled(void)
if (test_bit(PMD_ORDER, &huge_anon_orders_inherit) &&
hugepage_global_enabled())
return true;
if (IS_ENABLED(CONFIG_SHMEM) && shmem_hpage_pmd_enabled())
return true;
return false;
}
mm/ksm.c
-1
View File
@@ -20,7 +20,6 @@
#include <linux/mman.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/sched/coredump.h>
#include <linux/sched/cputime.h>
#include <linux/rwsem.h>
#include <linux/pagemap.h>
mm/madvise.c
+38 -21
View File
@@ -1208,7 +1208,8 @@ madvise_behavior_valid(int behavior)
}
}
static bool process_madvise_behavior_valid(int behavior)
/* Can we invoke process_madvise() on a remote mm for the specified behavior? */
static bool process_madvise_remote_valid(int behavior)
{
switch (behavior) {
case MADV_COLD:
@@ -1477,6 +1478,28 @@ SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
return do_madvise(current->mm, start, len_in, behavior);
}
/* Perform an madvise operation over a vector of addresses and lengths. */
static ssize_t vector_madvise(struct mm_struct *mm, struct iov_iter *iter,
int behavior)
{
ssize_t ret = 0;
size_t total_len;
total_len = iov_iter_count(iter);
while (iov_iter_count(iter)) {
ret = do_madvise(mm, (unsigned long)iter_iov_addr(iter),
iter_iov_len(iter), behavior);
if (ret < 0)
break;
iov_iter_advance(iter, iter_iov_len(iter));
}
ret = (total_len - iov_iter_count(iter)) ? : ret;
return ret;
}
SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec,
size_t, vlen, int, behavior, unsigned int, flags)
{
@@ -1486,7 +1509,6 @@ SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec,
struct iov_iter iter;
struct task_struct *task;
struct mm_struct *mm;
size_t total_len;
unsigned int f_flags;
if (flags != 0) {
@@ -1504,38 +1526,33 @@ SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec,
goto free_iov;
}
if (!process_madvise_behavior_valid(behavior)) {
ret = -EINVAL;
goto release_task;
}
/* Require PTRACE_MODE_READ to avoid leaking ASLR metadata. */
mm = mm_access(task, PTRACE_MODE_READ_FSCREDS);
if (IS_ERR_OR_NULL(mm)) {
ret = IS_ERR(mm) ? PTR_ERR(mm) : -ESRCH;
if (IS_ERR(mm)) {
ret = PTR_ERR(mm);
goto release_task;
}
/*
* We need only perform this check if we are attempting to manipulate a
* remote process's address space.
*/
if (mm != current->mm && !process_madvise_remote_valid(behavior)) {
ret = -EINVAL;
goto release_mm;
}
/*
* Require CAP_SYS_NICE for influencing process performance. Note that
* only non-destructive hints are currently supported.
* only non-destructive hints are currently supported for remote
* processes.
*/
if (mm != current->mm && !capable(CAP_SYS_NICE)) {
ret = -EPERM;
goto release_mm;
}
total_len = iov_iter_count(&iter);
while (iov_iter_count(&iter)) {
ret = do_madvise(mm, (unsigned long)iter_iov_addr(&iter),
iter_iov_len(&iter), behavior);
if (ret < 0)
break;
iov_iter_advance(&iter, iter_iov_len(&iter));
}
ret = (total_len - iov_iter_count(&iter)) ? : ret;
ret = vector_madvise(mm, &iter, behavior);
release_mm:
mmput(mm);
mm/memcontrol.c
+2
View File
@@ -418,6 +418,8 @@ static const unsigned int memcg_vm_event_stat[] = {
PGPGIN,
PGPGOUT,
#endif
PSWPIN,
PSWPOUT,
PGSCAN_KSWAPD,
PGSCAN_DIRECT,
PGSCAN_KHUGEPAGED,
mm/memory.c
-1
View File
@@ -44,7 +44,6 @@
#include <linux/mm.h>
#include <linux/mm_inline.h>
#include <linux/sched/mm.h>
#include <linux/sched/coredump.h>
#include <linux/sched/numa_balancing.h>
#include <linux/sched/task.h>
#include <linux/hugetlb.h>
mm/mempolicy.c
+1 -1
View File
@@ -8,7 +8,7 @@
* NUMA policy allows the user to give hints in which node(s) memory should
* be allocated.
*
* Support four policies per VMA and per process:
* Support six policies per VMA and per process:
*
* The VMA policy has priority over the process policy for a page fault.
*
mm/oom_kill.c
-1
View File
@@ -24,7 +24,6 @@
#include <linux/gfp.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/sched/coredump.h>
#include <linux/sched/task.h>
#include <linux/sched/debug.h>
#include <linux/swap.h>
mm/page_io.c
+4
View File
@@ -298,6 +298,7 @@ static inline void count_swpout_vm_event(struct folio *folio)
}
count_mthp_stat(folio_order(folio), MTHP_STAT_SWPOUT);
#endif
count_memcg_folio_events(folio, PSWPOUT, folio_nr_pages(folio));
count_vm_events(PSWPOUT, folio_nr_pages(folio));
}
@@ -493,6 +494,7 @@ static void sio_read_complete(struct kiocb *iocb, long ret)
for (p = 0; p < sio->pages; p++) {
struct folio *folio = page_folio(sio->bvec[p].bv_page);
count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio));
folio_mark_uptodate(folio);
folio_unlock(folio);
}
@@ -586,6 +588,7 @@ static void swap_read_folio_bdev_sync(struct folio *folio,
* attempt to access it in the page fault retry time check.
*/
get_task_struct(current);
count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio));
count_vm_events(PSWPIN, folio_nr_pages(folio));
submit_bio_wait(&bio);
__end_swap_bio_read(&bio);
@@ -601,6 +604,7 @@ static void swap_read_folio_bdev_async(struct folio *folio,
bio->bi_iter.bi_sector = swap_folio_sector(folio);
bio->bi_end_io = end_swap_bio_read;
bio_add_folio_nofail(bio, folio, folio_size(folio), 0);
count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio));
count_vm_events(PSWPIN, folio_nr_pages(folio));
submit_bio(bio);
}
mm/pgtable-generic.c
+48
View File
@@ -317,6 +317,31 @@ pte_t *pte_offset_map_nolock(struct mm_struct *mm, pmd_t *pmd,
return pte;
}
pte_t *pte_offset_map_ro_nolock(struct mm_struct *mm, pmd_t *pmd,
unsigned long addr, spinlock_t **ptlp)
{
pmd_t pmdval;
pte_t *pte;
pte = __pte_offset_map(pmd, addr, &pmdval);
if (likely(pte))
*ptlp = pte_lockptr(mm, &pmdval);
return pte;
}
pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd,
unsigned long addr, pmd_t *pmdvalp,
spinlock_t **ptlp)
{
pte_t *pte;
VM_WARN_ON_ONCE(!pmdvalp);
pte = __pte_offset_map(pmd, addr, pmdvalp);
if (likely(pte))
*ptlp = pte_lockptr(mm, pmdvalp);
return pte;
}
/*
* pte_offset_map_lock(mm, pmd, addr, ptlp), and its internal implementation
* __pte_offset_map_lock() below, is usually called with the pmd pointer for
@@ -356,6 +381,29 @@ pte_t *pte_offset_map_nolock(struct mm_struct *mm, pmd_t *pmd,
* recheck *pmd once the lock is taken; in practice, no callsite needs that -
* either the mmap_lock for write, or pte_same() check on contents, is enough.
*
* pte_offset_map_ro_nolock(mm, pmd, addr, ptlp), above, is like pte_offset_map();
* but when successful, it also outputs a pointer to the spinlock in ptlp - as
* pte_offset_map_lock() does, but in this case without locking it. This helps
* the caller to avoid a later pte_lockptr(mm, *pmd), which might by that time
* act on a changed *pmd: pte_offset_map_ro_nolock() provides the correct spinlock
* pointer for the page table that it returns. Even after grabbing the spinlock,
* we might be looking either at a page table that is still mapped or one that
* was unmapped and is about to get freed. But for R/O access this is sufficient.
* So it is only applicable for read-only cases where any modification operations
* to the page table are not allowed even if the corresponding spinlock is held
* afterwards.
*
* pte_offset_map_rw_nolock(mm, pmd, addr, pmdvalp, ptlp), above, is like
* pte_offset_map_ro_nolock(); but when successful, it also outputs the pdmval.
* It is applicable for may-write cases where any modification operations to the
* page table may happen after the corresponding spinlock is held afterwards.
* But the users should make sure the page table is stable like checking pte_same()
* or checking pmd_same() by using the output pmdval before performing the write
* operations.
*
* Note: "RO" / "RW" expresses the intended semantics, not that the *kmap* will
* be read-only/read-write protected.
*
* Note that free_pgtables(), used after unmapping detached vmas, or when
* exiting the whole mm, does not take page table lock before freeing a page
* table, and may not use RCU at all: "outsiders" like khugepaged should avoid
mm/process_vm_access.c
+2 -2
View File
@@ -201,8 +201,8 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter,
}
mm = mm_access(task, PTRACE_MODE_ATTACH_REALCREDS);
if (!mm || IS_ERR(mm)) {
rc = IS_ERR(mm) ? PTR_ERR(mm) : -ESRCH;
if (IS_ERR(mm)) {
rc = PTR_ERR(mm);
/*
* Explicitly map EACCES to EPERM as EPERM is a more
* appropriate error code for process_vw_readv/writev
mm/shmem.c
+27 -2
View File
@@ -1658,6 +1658,23 @@ static gfp_t limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp)
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
bool shmem_hpage_pmd_enabled(void)
{
if (shmem_huge == SHMEM_HUGE_DENY)
return false;
if (test_bit(HPAGE_PMD_ORDER, &huge_shmem_orders_always))
return true;
if (test_bit(HPAGE_PMD_ORDER, &huge_shmem_orders_madvise))
return true;
if (test_bit(HPAGE_PMD_ORDER, &huge_shmem_orders_within_size))
return true;
if (test_bit(HPAGE_PMD_ORDER, &huge_shmem_orders_inherit) &&
shmem_huge != SHMEM_HUGE_NEVER)
return true;
return false;
}
unsigned long shmem_allowable_huge_orders(struct inode *inode,
struct vm_area_struct *vma, pgoff_t index,
loff_t write_end, bool shmem_huge_force)
@@ -5267,7 +5284,7 @@ static ssize_t shmem_enabled_store(struct kobject *kobj,
struct kobj_attribute *attr, const char *buf, size_t count)
{
char tmp[16];
int huge;
int huge, err;
if (count + 1 > sizeof(tmp))
return -EINVAL;
@@ -5291,7 +5308,9 @@ static ssize_t shmem_enabled_store(struct kobject *kobj,
shmem_huge = huge;
if (shmem_huge > SHMEM_HUGE_DENY)
SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
return count;
err = start_stop_khugepaged();
return err ? err : count;
}
struct kobj_attribute shmem_enabled_attr = __ATTR_RW(shmem_enabled);
@@ -5368,6 +5387,12 @@ static ssize_t thpsize_shmem_enabled_store(struct kobject *kobj,
ret = -EINVAL;
}
if (ret > 0) {
int err = start_stop_khugepaged();
if (err)
ret = err;
}
return ret;
}
mm/vmalloc.c
+26 -8
View File
@@ -2182,6 +2182,25 @@ decay_va_pool_node(struct vmap_node *vn, bool full_decay)
reclaim_list_global(&decay_list);
}
static void
kasan_release_vmalloc_node(struct vmap_node *vn)
{
struct vmap_area *va;
unsigned long start, end;
start = list_first_entry(&vn->purge_list, struct vmap_area, list)->va_start;
end = list_last_entry(&vn->purge_list, struct vmap_area, list)->va_end;
list_for_each_entry(va, &vn->purge_list, list) {
if (is_vmalloc_or_module_addr((void *) va->va_start))
kasan_release_vmalloc(va->va_start, va->va_end,
va->va_start, va->va_end,
KASAN_VMALLOC_PAGE_RANGE);
}
kasan_release_vmalloc(start, end, start, end, KASAN_VMALLOC_TLB_FLUSH);
}
static void purge_vmap_node(struct work_struct *work)
{
struct vmap_node *vn = container_of(work,
@@ -2190,20 +2209,17 @@ static void purge_vmap_node(struct work_struct *work)
struct vmap_area *va, *n_va;
LIST_HEAD(local_list);
if (IS_ENABLED(CONFIG_KASAN_VMALLOC))
kasan_release_vmalloc_node(vn);
vn->nr_purged = 0;
list_for_each_entry_safe(va, n_va, &vn->purge_list, list) {
unsigned long nr = va_size(va) >> PAGE_SHIFT;
unsigned long orig_start = va->va_start;
unsigned long orig_end = va->va_end;
unsigned int vn_id = decode_vn_id(va->flags);
list_del_init(&va->list);
if (is_vmalloc_or_module_addr((void *)orig_start))
kasan_release_vmalloc(orig_start, orig_end,
va->va_start, va->va_end);
nr_purged_pages += nr;
vn->nr_purged++;
@@ -4784,7 +4800,8 @@ recovery:
&free_vmap_area_list);
if (va)
kasan_release_vmalloc(orig_start, orig_end,
va->va_start, va->va_end);
va->va_start, va->va_end,
KASAN_VMALLOC_PAGE_RANGE | KASAN_VMALLOC_TLB_FLUSH);
vas[area] = NULL;
}
@@ -4834,7 +4851,8 @@ err_free_shadow:
&free_vmap_area_list);
if (va)
kasan_release_vmalloc(orig_start, orig_end,
va->va_start, va->va_end);
va->va_start, va->va_end,
KASAN_VMALLOC_PAGE_RANGE | KASAN_VMALLOC_TLB_FLUSH);
vas[area] = NULL;
kfree(vms[area]);
}
mm/vmscan.c
+2 -1
View File
@@ -5998,7 +5998,8 @@ again:
set_bit(PGDAT_WRITEBACK, &pgdat->flags);
/* Allow kswapd to start writing pages during reclaim.*/
if (sc->nr.unqueued_dirty == sc->nr.file_taken)
if (sc->nr.unqueued_dirty &&
sc->nr.unqueued_dirty == sc->nr.file_taken)
set_bit(PGDAT_DIRTY, &pgdat->flags);
/*
tools/testing/selftests/mm/.gitignore
+3
View File
@@ -51,3 +51,6 @@ hugetlb_madv_vs_map
mseal_test
seal_elf
droppable
hugetlb_dio
pkey_sighandler_tests_32
pkey_sighandler_tests_64
tools/testing/selftests/mm/hugetlb_fault_after_madv.c
+42 -6
View File
@@ -5,20 +5,36 @@
#include <sys/mman.h>
#include <sys/types.h>
#include <unistd.h>
#include <setjmp.h>
#include <signal.h>
#include "vm_util.h"
#include "../kselftest.h"
#define MMAP_SIZE (1 << 21)
#define INLOOP_ITER 100
char *huge_ptr;
static char *huge_ptr;
static size_t huge_page_size;
static sigjmp_buf sigbuf;
static bool sigbus_triggered;
static void signal_handler(int signal)
{
if (signal == SIGBUS) {
sigbus_triggered = true;
siglongjmp(sigbuf, 1);
}
}
/* Touch the memory while it is being madvised() */
void *touch(void *unused)
{
char *ptr = (char *)huge_ptr;
if (sigsetjmp(sigbuf, 1))
return NULL;
for (int i = 0; i < INLOOP_ITER; i++)
ptr[0] = '.';
@@ -30,7 +46,7 @@ void *madv(void *unused)
usleep(rand() % 10);
for (int i = 0; i < INLOOP_ITER; i++)
madvise(huge_ptr, MMAP_SIZE, MADV_DONTNEED);
madvise(huge_ptr, huge_page_size, MADV_DONTNEED);
return NULL;
}
@@ -44,9 +60,23 @@ int main(void)
* interactions
*/
int max = 10000;
int err;
ksft_print_header();
ksft_set_plan(1);
srand(getpid());
if (signal(SIGBUS, signal_handler) == SIG_ERR)
ksft_exit_skip("Could not register signal handler.");
huge_page_size = default_huge_page_size();
if (!huge_page_size)
ksft_exit_skip("Could not detect default hugetlb page size.");
ksft_print_msg("[INFO] detected default hugetlb page size: %zu KiB\n",
huge_page_size / 1024);
free_hugepages = get_free_hugepages();
if (free_hugepages != 1) {
ksft_exit_skip("This test needs one and only one page to execute. Got %lu\n",
@@ -54,7 +84,7 @@ int main(void)
}
while (max--) {
huge_ptr = mmap(NULL, MMAP_SIZE, PROT_READ | PROT_WRITE,
huge_ptr = mmap(NULL, huge_page_size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
-1, 0);
@@ -66,8 +96,14 @@ int main(void)
pthread_join(thread1, NULL);
pthread_join(thread2, NULL);
munmap(huge_ptr, MMAP_SIZE);
munmap(huge_ptr, huge_page_size);
}
return KSFT_PASS;
ksft_test_result(!sigbus_triggered, "SIGBUS behavior\n");
err = ksft_get_fail_cnt();
if (err)
ksft_exit_fail_msg("%d out of %d tests failed\n",
err, ksft_test_num());
ksft_exit_pass();
}