diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 212fc498caab..fdf7e8362132 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -344,7 +344,7 @@ show_map_vma(struct seq_file *m, struct vm_area_struct *vma) } start = vma->vm_start; - end = vma->vm_end; + end = VMA_PAD_START(vma); __fold_filemap_fixup_entry(&((struct proc_maps_private *)m->private)->iter, &end); @@ -366,13 +366,12 @@ show_map_vma(struct seq_file *m, struct vm_area_struct *vma) static int show_map(struct seq_file *m, void *v) { - struct vm_area_struct *pad_vma = get_pad_vma(v); - struct vm_area_struct *vma = get_data_vma(v); + struct vm_area_struct *vma = v; if (vma_pages(vma)) show_map_vma(m, vma); - show_map_pad_vma(vma, pad_vma, m, show_map_vma, false); + show_map_pad_vma(vma, m, show_map_vma, false); return 0; } @@ -1078,9 +1077,10 @@ static void smap_gather_stats(struct vm_area_struct *vma, struct mem_size_stats *mss, unsigned long start) { const struct mm_walk_ops *ops = &smaps_walk_ops; + unsigned long end = VMA_PAD_START(vma); /* Invalid start */ - if (start >= vma->vm_end) + if (start >= end) return; if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) { @@ -1097,7 +1097,15 @@ static void smap_gather_stats(struct vm_area_struct *vma, unsigned long shmem_swapped = shmem_swap_usage(vma); if (!start && (!shmem_swapped || (vma->vm_flags & VM_SHARED) || - !(vma->vm_flags & VM_WRITE))) { + !(vma->vm_flags & VM_WRITE)) && + /* + * Only if we don't have padding can we use the fast path + * shmem_inode_info->swapped for shmem_swapped. + * + * Else we'll walk the page table to calculate + * shmem_swapped, (excluding the padding region). + */ + end == vma->vm_end) { mss->swap += shmem_swapped; } else { ops = &smaps_shmem_walk_ops; @@ -1106,9 +1114,9 @@ static void smap_gather_stats(struct vm_area_struct *vma, /* mmap_lock is held in m_start */ if (!start) - walk_page_vma(vma, ops, mss); + walk_page_range(vma->vm_mm, vma->vm_start, end, ops, mss); else - walk_page_range(vma->vm_mm, start, vma->vm_end, ops, mss); + walk_page_range(vma->vm_mm, start, end, ops, mss); } #define SEQ_PUT_DEC(str, val) \ @@ -1159,8 +1167,7 @@ static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss, static int show_smap(struct seq_file *m, void *v) { - struct vm_area_struct *pad_vma = get_pad_vma(v); - struct vm_area_struct *vma = get_data_vma(v); + struct vm_area_struct *vma = v; struct mem_size_stats mss = {}; if (!vma_pages(vma)) @@ -1170,7 +1177,7 @@ static int show_smap(struct seq_file *m, void *v) show_map_vma(m, vma); - SEQ_PUT_DEC("Size: ", vma->vm_end - vma->vm_start); + SEQ_PUT_DEC("Size: ", VMA_PAD_START(vma) - vma->vm_start); SEQ_PUT_DEC(" kB\nKernelPageSize: ", vma_kernel_pagesize(vma)); SEQ_PUT_DEC(" kB\nMMUPageSize: ", vma_mmu_pagesize(vma)); seq_puts(m, " kB\n"); @@ -1186,7 +1193,7 @@ static int show_smap(struct seq_file *m, void *v) show_smap_vma_flags(m, vma); show_pad: - show_map_pad_vma(vma, pad_vma, m, show_smap, true); + show_map_pad_vma(vma, m, show_smap, true); return 0; } diff --git a/include/linux/pgsize_migration.h b/include/linux/pgsize_migration.h index 48672dbc84e9..359c1807ff1d 100644 --- a/include/linux/pgsize_migration.h +++ b/include/linux/pgsize_migration.h @@ -26,12 +26,7 @@ extern unsigned long vma_pad_pages(struct vm_area_struct *vma); extern void madvise_vma_pad_pages(struct vm_area_struct *vma, unsigned long start, unsigned long end); -extern struct vm_area_struct *get_pad_vma(struct vm_area_struct *vma); - -extern struct vm_area_struct *get_data_vma(struct vm_area_struct *vma); - extern void show_map_pad_vma(struct vm_area_struct *vma, - struct vm_area_struct *pad, struct seq_file *m, void *func, bool smaps); extern void split_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *new, @@ -57,18 +52,7 @@ static inline void madvise_vma_pad_pages(struct vm_area_struct *vma, { } -static inline struct vm_area_struct *get_pad_vma(struct vm_area_struct *vma) -{ - return NULL; -} - -static inline struct vm_area_struct *get_data_vma(struct vm_area_struct *vma) -{ - return vma; -} - static inline void show_map_pad_vma(struct vm_area_struct *vma, - struct vm_area_struct *pad, struct seq_file *m, void *func, bool smaps) { } diff --git a/mm/pgsize_migration.c b/mm/pgsize_migration.c index 623a50e965a8..3b020924a7a9 100644 --- a/mm/pgsize_migration.c +++ b/mm/pgsize_migration.c @@ -271,10 +271,10 @@ static const struct vm_operations_struct pad_vma_ops = { }; /* - * Returns a new VMA representing the padding in @vma, if no padding - * in @vma returns NULL. + * Returns a new VMA representing the padding in @vma; + * returns NULL if no padding in @vma or allocation failed. */ -struct vm_area_struct *get_pad_vma(struct vm_area_struct *vma) +static struct vm_area_struct *get_pad_vma(struct vm_area_struct *vma) { struct vm_area_struct *pad; @@ -282,6 +282,10 @@ struct vm_area_struct *get_pad_vma(struct vm_area_struct *vma) return NULL; pad = kzalloc(sizeof(struct vm_area_struct), GFP_KERNEL); + if (!pad) { + pr_warn("Page size migration: Failed to allocate padding VMA"); + return NULL; + } memcpy(pad, vma, sizeof(struct vm_area_struct)); @@ -306,34 +310,14 @@ struct vm_area_struct *get_pad_vma(struct vm_area_struct *vma) return pad; } -/* - * Returns a new VMA exclusing the padding from @vma; if no padding in - * @vma returns @vma. - */ -struct vm_area_struct *get_data_vma(struct vm_area_struct *vma) -{ - struct vm_area_struct *data; - - if (!is_pgsize_migration_enabled() || !(vma->vm_flags & VM_PAD_MASK)) - return vma; - - data = kzalloc(sizeof(struct vm_area_struct), GFP_KERNEL); - - memcpy(data, vma, sizeof(struct vm_area_struct)); - - /* Adjust the end to the start of the padding section */ - data->vm_end = VMA_PAD_START(data); - - return data; -} - /* * Calls the show_pad_vma_fn on the @pad VMA, and frees the copies of @vma * and @pad. */ -void show_map_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *pad, - struct seq_file *m, void *func, bool smaps) +void show_map_pad_vma(struct vm_area_struct *vma, struct seq_file *m, + void *func, bool smaps) { + struct vm_area_struct *pad = get_pad_vma(vma); if (!pad) return; @@ -349,13 +333,21 @@ void show_map_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *pad, */ BUG_ON(!vma); + /* The pad VMA should be anonymous. */ + BUG_ON(pad->vm_file); + + /* The pad VMA should be PROT_NONE. */ + BUG_ON(pad->vm_flags & (VM_READ|VM_WRITE|VM_EXEC)); + + /* The pad VMA itself cannot have padding; infinite recursion */ + BUG_ON(pad->vm_flags & VM_PAD_MASK); + if (smaps) ((show_pad_smaps_fn)func)(m, pad); else ((show_pad_maps_fn)func)(m, pad); kfree(pad); - kfree(vma); } /*