diff --git a/drivers/android/binder.c b/drivers/android/binder.c index 1479525e2913..77d5012b02b9 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -5575,7 +5575,7 @@ static void binder_free_proc(struct binder_proc *proc) binder_stats_deleted(BINDER_STAT_PROC); dbitmap_free(&proc->dmap); trace_android_vh_binder_free_proc(proc); - kfree(proc); + kfree(proc_wrapper(proc)); } static void binder_free_thread(struct binder_thread *thread) @@ -6327,6 +6327,7 @@ static int binder_mmap(struct file *filp, struct vm_area_struct *vma) static int binder_open(struct inode *nodp, struct file *filp) { + struct binder_proc_wrap *proc_wrap; struct binder_proc *proc, *itr; struct binder_device *binder_dev; struct binderfs_info *info; @@ -6336,9 +6337,10 @@ static int binder_open(struct inode *nodp, struct file *filp) binder_debug(BINDER_DEBUG_OPEN_CLOSE, "%s: %d:%d\n", __func__, current->group_leader->pid, current->pid); - proc = kzalloc(sizeof(*proc), GFP_KERNEL); - if (proc == NULL) + proc_wrap = kzalloc(sizeof(*proc_wrap), GFP_KERNEL); + if (proc_wrap == NULL) return -ENOMEM; + proc = &proc_wrap->proc; dbitmap_init(&proc->dmap); spin_lock_init(&proc->inner_lock); diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c index 5918d161da58..f4b4f62484d3 100644 --- a/drivers/android/binder_alloc.c +++ b/drivers/android/binder_alloc.c @@ -23,10 +23,13 @@ #include #include #include -#include "binder_alloc.h" +#include "binder_internal.h" #include "binder_trace.h" #include +ANDROID_KABI_DECLONLY(binder_node); +ANDROID_KABI_DECLONLY(binder_transaction); + struct list_lru binder_freelist; static DEFINE_MUTEX(binder_alloc_mmap_lock); @@ -170,9 +173,9 @@ struct binder_buffer *binder_alloc_prepare_to_free(struct binder_alloc *alloc, { struct binder_buffer *buffer; - mutex_lock(&alloc->mutex); + mutex_lock(&alloc_to_wrap(alloc)->mutex); buffer = binder_alloc_prepare_to_free_locked(alloc, user_ptr); - mutex_unlock(&alloc->mutex); + mutex_unlock(&alloc_to_wrap(alloc)->mutex); return buffer; } @@ -182,14 +185,14 @@ binder_set_installed_page(struct binder_alloc *alloc, struct page *page) { /* Pairs with acquire in binder_get_installed_page() */ - smp_store_release(&alloc->pages[index], page); + smp_store_release(&alloc_to_wrap(alloc)->pages[index], page); } static inline struct page * binder_get_installed_page(struct binder_alloc *alloc, unsigned long index) { /* Pairs with release in binder_set_installed_page() */ - return smp_load_acquire(&alloc->pages[index]); + return smp_load_acquire(&alloc_to_wrap(alloc)->pages[index]); } static void binder_lru_freelist_add(struct binder_alloc *alloc, @@ -225,13 +228,13 @@ static inline void binder_alloc_set_mapped(struct binder_alloc *alloc, bool state) { /* pairs with smp_load_acquire in binder_alloc_is_mapped() */ - smp_store_release(&alloc->mapped, state); + smp_store_release(&alloc_to_wrap(alloc)->mapped, state); } static inline bool binder_alloc_is_mapped(struct binder_alloc *alloc) { /* pairs with smp_store_release in binder_alloc_set_mapped() */ - return smp_load_acquire(&alloc->mapped); + return smp_load_acquire(&alloc_to_wrap(alloc)->mapped); } static struct page *binder_page_lookup(struct binder_alloc *alloc, @@ -690,10 +693,10 @@ struct binder_buffer *binder_alloc_new_buf(struct binder_alloc *alloc, if (!next) return ERR_PTR(-ENOMEM); - mutex_lock(&alloc->mutex); + mutex_lock(&alloc_to_wrap(alloc)->mutex); buffer = binder_alloc_new_buf_locked(alloc, next, size, is_async); if (IS_ERR(buffer)) { - mutex_unlock(&alloc->mutex); + mutex_unlock(&alloc_to_wrap(alloc)->mutex); goto out; } @@ -701,7 +704,7 @@ struct binder_buffer *binder_alloc_new_buf(struct binder_alloc *alloc, buffer->offsets_size = offsets_size; buffer->extra_buffers_size = extra_buffers_size; buffer->pid = current->tgid; - mutex_unlock(&alloc->mutex); + mutex_unlock(&alloc_to_wrap(alloc)->mutex); ret = binder_install_buffer_pages(alloc, buffer, size); if (ret) { @@ -833,7 +836,7 @@ static struct page *binder_alloc_get_page(struct binder_alloc *alloc, *pgoffp = pgoff; - return alloc->pages[index]; + return alloc_to_wrap(alloc)->pages[index]; } /** @@ -885,9 +888,9 @@ void binder_alloc_free_buf(struct binder_alloc *alloc, binder_alloc_clear_buf(alloc, buffer); buffer->clear_on_free = false; } - mutex_lock(&alloc->mutex); + mutex_lock(&alloc_to_wrap(alloc)->mutex); binder_free_buf_locked(alloc, buffer); - mutex_unlock(&alloc->mutex); + mutex_unlock(&alloc_to_wrap(alloc)->mutex); } /** @@ -928,10 +931,10 @@ int binder_alloc_mmap_handler(struct binder_alloc *alloc, alloc->buffer = vma->vm_start; - alloc->pages = kvcalloc(alloc->buffer_size / PAGE_SIZE, - sizeof(alloc->pages[0]), + alloc_to_wrap(alloc)->pages = kvcalloc(alloc->buffer_size / PAGE_SIZE, + sizeof(alloc_to_wrap(alloc)->pages[0]), GFP_KERNEL); - if (!alloc->pages) { + if (!alloc_to_wrap(alloc)->pages) { ret = -ENOMEM; failure_string = "alloc page array"; goto err_alloc_pages_failed; @@ -956,8 +959,8 @@ int binder_alloc_mmap_handler(struct binder_alloc *alloc, return 0; err_alloc_buf_struct_failed: - kvfree(alloc->pages); - alloc->pages = NULL; + kvfree(alloc_to_wrap(alloc)->pages); + alloc_to_wrap(alloc)->pages = NULL; err_alloc_pages_failed: alloc->buffer = 0; mutex_lock(&binder_alloc_mmap_lock); @@ -980,8 +983,8 @@ void binder_alloc_deferred_release(struct binder_alloc *alloc) struct binder_buffer *buffer; buffers = 0; - mutex_lock(&alloc->mutex); - BUG_ON(alloc->mapped); + mutex_lock(&alloc_to_wrap(alloc)->mutex); + BUG_ON(alloc_to_wrap(alloc)->mapped); while ((n = rb_first(&alloc->allocated_buffers))) { buffer = rb_entry(n, struct binder_buffer, rb_node); @@ -1008,7 +1011,7 @@ void binder_alloc_deferred_release(struct binder_alloc *alloc) } page_count = 0; - if (alloc->pages) { + if (alloc_to_wrap(alloc)->pages) { int i; for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) { @@ -1031,8 +1034,8 @@ void binder_alloc_deferred_release(struct binder_alloc *alloc) page_count++; } } - mutex_unlock(&alloc->mutex); - kvfree(alloc->pages); + mutex_unlock(&alloc_to_wrap(alloc)->mutex); + kvfree(alloc_to_wrap(alloc)->pages); if (alloc->mm) mmdrop(alloc->mm); @@ -1055,7 +1058,7 @@ void binder_alloc_print_allocated(struct seq_file *m, struct binder_buffer *buffer; struct rb_node *n; - mutex_lock(&alloc->mutex); + mutex_lock(&alloc_to_wrap(alloc)->mutex); for (n = rb_first(&alloc->allocated_buffers); n; n = rb_next(n)) { buffer = rb_entry(n, struct binder_buffer, rb_node); seq_printf(m, " buffer %d: %lx size %zd:%zd:%zd %s\n", @@ -1065,7 +1068,7 @@ void binder_alloc_print_allocated(struct seq_file *m, buffer->extra_buffers_size, buffer->transaction ? "active" : "delivered"); } - mutex_unlock(&alloc->mutex); + mutex_unlock(&alloc_to_wrap(alloc)->mutex); } /** @@ -1082,7 +1085,7 @@ void binder_alloc_print_pages(struct seq_file *m, int lru = 0; int free = 0; - mutex_lock(&alloc->mutex); + mutex_lock(&alloc_to_wrap(alloc)->mutex); /* * Make sure the binder_alloc is fully initialized, otherwise we might * read inconsistent state. @@ -1098,7 +1101,7 @@ void binder_alloc_print_pages(struct seq_file *m, lru++; } } - mutex_unlock(&alloc->mutex); + mutex_unlock(&alloc_to_wrap(alloc)->mutex); seq_printf(m, " pages: %d:%d:%d\n", active, lru, free); seq_printf(m, " pages high watermark: %zu\n", alloc->pages_high); } @@ -1114,10 +1117,10 @@ int binder_alloc_get_allocated_count(struct binder_alloc *alloc) struct rb_node *n; int count = 0; - mutex_lock(&alloc->mutex); + mutex_lock(&alloc_to_wrap(alloc)->mutex); for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n)) count++; - mutex_unlock(&alloc->mutex); + mutex_unlock(&alloc_to_wrap(alloc)->mutex); return count; } @@ -1175,7 +1178,7 @@ enum lru_status binder_alloc_free_page(struct list_head *item, vma = vma_lookup(mm, page_addr); } - if (!mutex_trylock(&alloc->mutex)) + if (!mutex_trylock(&alloc_to_wrap(alloc)->mutex)) goto err_get_alloc_mutex_failed; /* @@ -1188,7 +1191,7 @@ enum lru_status binder_alloc_free_page(struct list_head *item, trace_binder_unmap_kernel_start(alloc, index); - page_to_free = alloc->pages[index]; + page_to_free = alloc_to_wrap(alloc)->pages[index]; binder_set_installed_page(alloc, index, NULL); trace_binder_unmap_kernel_end(alloc, index); @@ -1204,7 +1207,7 @@ enum lru_status binder_alloc_free_page(struct list_head *item, trace_binder_unmap_user_end(alloc, index); } - mutex_unlock(&alloc->mutex); + mutex_unlock(&alloc_to_wrap(alloc)->mutex); if (mm_locked) mmap_read_unlock(mm); else @@ -1216,7 +1219,7 @@ enum lru_status binder_alloc_free_page(struct list_head *item, return LRU_REMOVED_RETRY; err_invalid_vma: - mutex_unlock(&alloc->mutex); + mutex_unlock(&alloc_to_wrap(alloc)->mutex); err_get_alloc_mutex_failed: if (mm_locked) mmap_read_unlock(mm); @@ -1255,7 +1258,7 @@ void binder_alloc_init(struct binder_alloc *alloc) alloc->pid = current->group_leader->pid; alloc->mm = current->mm; mmgrab(alloc->mm); - mutex_init(&alloc->mutex); + mutex_init(&alloc_to_wrap(alloc)->mutex); INIT_LIST_HEAD(&alloc->buffers); } diff --git a/drivers/android/binder_alloc.h b/drivers/android/binder_alloc.h index c83658504d54..0eab381c196b 100644 --- a/drivers/android/binder_alloc.h +++ b/drivers/android/binder_alloc.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -70,6 +71,12 @@ struct binder_shrinker_mdata { unsigned long page_index; }; +struct binder_lru_page { + struct list_head lru; + struct page *page_ptr; + struct binder_alloc *alloc; +}; + static inline struct list_head *page_to_lru(struct page *p) { struct binder_shrinker_mdata *mdata; @@ -81,7 +88,9 @@ static inline struct list_head *page_to_lru(struct page *p) /** * struct binder_alloc - per-binder proc state for binder allocator - * @mutex: protects binder_alloc fields + * @lock: protects binder_alloc fields + * @vma: vm_area_struct passed to mmap_handler + * (invariant after mmap) * @mm: copy of task->mm (invariant after open) * @buffer: base of per-proc address space mapped via mmap * @buffers: list of all buffers for this proc @@ -90,12 +99,10 @@ static inline struct list_head *page_to_lru(struct page *p) * @allocated_buffers: rb tree of allocated buffers sorted by address * @free_async_space: VA space available for async buffers. This is * initialized at mmap time to 1/2 the full VA space - * @pages: array of struct page * + * @pages: array of binder_lru_page * @buffer_size: size of address space specified via mmap * @pid: pid for associated binder_proc (invariant after init) * @pages_high: high watermark of offset in @pages - * @mapped: whether the vm area is mapped, each binder instance is - * allowed a single mapping throughout its lifetime * @oneway_spam_detected: %true if oneway spam detection fired, clear that * flag once the async buffer has returned to a healthy state * @@ -105,18 +112,18 @@ static inline struct list_head *page_to_lru(struct page *p) * struct binder_buffer objects used to track the user buffers */ struct binder_alloc { - struct mutex mutex; + spinlock_t lock; + struct vm_area_struct *vma; struct mm_struct *mm; unsigned long buffer; struct list_head buffers; struct rb_root free_buffers; struct rb_root allocated_buffers; size_t free_async_space; - struct page **pages; + struct binder_lru_page *pages; size_t buffer_size; int pid; size_t pages_high; - bool mapped; bool oneway_spam_detected; ANDROID_OEM_DATA(1); }; @@ -152,23 +159,6 @@ void binder_alloc_print_allocated(struct seq_file *m, void binder_alloc_print_pages(struct seq_file *m, struct binder_alloc *alloc); -/** - * binder_alloc_get_free_async_space() - get free space available for async - * @alloc: binder_alloc for this proc - * - * Return: the bytes remaining in the address-space for async transactions - */ -static inline size_t -binder_alloc_get_free_async_space(struct binder_alloc *alloc) -{ - size_t free_async_space; - - mutex_lock(&alloc->mutex); - free_async_space = alloc->free_async_space; - mutex_unlock(&alloc->mutex); - return free_async_space; -} - unsigned long binder_alloc_copy_user_to_buffer(struct binder_alloc *alloc, struct binder_buffer *buffer, diff --git a/drivers/android/binder_alloc_selftest.c b/drivers/android/binder_alloc_selftest.c index 6a64847a8555..86592248c313 100644 --- a/drivers/android/binder_alloc_selftest.c +++ b/drivers/android/binder_alloc_selftest.c @@ -10,7 +10,7 @@ #include #include -#include "binder_alloc.h" +#include "binder_internal.h" #define BUFFER_NUM 5 #define BUFFER_MIN_SIZE (PAGE_SIZE / 8) @@ -105,10 +105,10 @@ static bool check_buffer_pages_allocated(struct binder_alloc *alloc, page_addr = buffer->user_data; for (; page_addr < end; page_addr += PAGE_SIZE) { page_index = (page_addr - alloc->buffer) / PAGE_SIZE; - if (!alloc->pages[page_index] || - !list_empty(page_to_lru(alloc->pages[page_index]))) { + if (!alloc_to_wrap(alloc)->pages[page_index] || + !list_empty(page_to_lru(alloc_to_wrap(alloc)->pages[page_index]))) { pr_err("expect alloc but is %s at page index %d\n", - alloc->pages[page_index] ? + alloc_to_wrap(alloc)->pages[page_index] ? "lru" : "free", page_index); return false; } @@ -148,10 +148,10 @@ static void binder_selftest_free_buf(struct binder_alloc *alloc, * if binder shrinker ran during binder_alloc_free_buf * calls above. */ - if (list_empty(page_to_lru(alloc->pages[i]))) { + if (list_empty(page_to_lru(alloc_to_wrap(alloc)->pages[i]))) { pr_err_size_seq(sizes, seq); pr_err("expect lru but is %s at page index %d\n", - alloc->pages[i] ? "alloc" : "free", i); + alloc_to_wrap(alloc)->pages[i] ? "alloc" : "free", i); binder_selftest_failures++; } } @@ -168,9 +168,9 @@ static void binder_selftest_free_page(struct binder_alloc *alloc) } for (i = 0; i < (alloc->buffer_size / PAGE_SIZE); i++) { - if (alloc->pages[i]) { + if (alloc_to_wrap(alloc)->pages[i]) { pr_err("expect free but is %s at page index %d\n", - list_empty(page_to_lru(alloc->pages[i])) ? + list_empty(page_to_lru(alloc_to_wrap(alloc)->pages[i])) ? "alloc" : "lru", i); binder_selftest_failures++; } @@ -291,7 +291,7 @@ void binder_selftest_alloc(struct binder_alloc *alloc) if (!binder_selftest_run) return; mutex_lock(&binder_selftest_lock); - if (!binder_selftest_run || !alloc->mapped) + if (!binder_selftest_run || !alloc_to_wrap(alloc)->mapped) goto done; pr_info("STARTED\n"); binder_selftest_alloc_offset(alloc, end_offset, 0); diff --git a/drivers/android/binder_internal.h b/drivers/android/binder_internal.h index 66c96ab9e7ae..667315ed2793 100644 --- a/drivers/android/binder_internal.h +++ b/drivers/android/binder_internal.h @@ -494,6 +494,56 @@ struct binder_proc { ANDROID_OEM_DATA(1); }; +/** + * struct binder_proc_wrap - wrapper to preserve KMI in binder_proc + * @proc: binder_proc being wrapped + * @mutex: protects binder_alloc fields + * @pages: array of struct page * + * @mapped: whether the vm area is mapped, each binderinstance + * is allowed a single mapping throughout its lifetime + */ +struct binder_proc_wrap { + struct binder_proc proc; + struct binder_alloc_wrap { + struct mutex mutex; + struct page **pages; + bool mapped; + } alloc; +}; + +static inline +struct binder_proc_wrap *proc_wrapper(struct binder_proc *proc) +{ + return container_of(proc, struct binder_proc_wrap, proc); +} + +static inline +struct binder_alloc_wrap *alloc_to_wrap(struct binder_alloc *alloc) +{ + struct binder_proc *proc; + + proc = container_of(alloc, struct binder_proc, alloc); + + return &proc_wrapper(proc)->alloc; +} + +/** + * binder_alloc_get_free_async_space() - get free space available for async + * @alloc: binder_alloc for this proc + * + * Return: the bytes remaining in the address-space for async transactions + */ +static inline size_t +binder_alloc_get_free_async_space(struct binder_alloc *alloc) +{ + size_t free_async_space; + + mutex_lock(&alloc_to_wrap(alloc)->mutex); + free_async_space = alloc->free_async_space; + mutex_unlock(&alloc_to_wrap(alloc)->mutex); + return free_async_space; +} + /** * struct binder_thread - binder thread bookkeeping * @proc: binder process for this thread