tmakin/tegra-linux-noble
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branches 'for-4.12/upstream' and 'for-4.12/klp-hybrid-consistency-model' into for-linus

Browse Source
This commit is contained in:
Jiri Kosina
2017-05-01 21:49:28 +02:00
parent 77f8f39a2e e679af627f
commit a0841609f6
30 changed files with 1537 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kernel/livepatch/Makefile
+1 -1
View File
@@ -1,3 +1,3 @@
obj-$(CONFIG_LIVEPATCH) += livepatch.o
livepatch-objs := core.o
livepatch-objs := core.o patch.o transition.o
kernel/livepatch/core.c
+164 -273
View File
@@ -24,61 +24,31 @@
#include <linux/kernel.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/ftrace.h>
#include <linux/list.h>
#include <linux/kallsyms.h>
#include <linux/livepatch.h>
#include <linux/elf.h>
#include <linux/moduleloader.h>
#include <linux/completion.h>
#include <asm/cacheflush.h>
/**
* struct klp_ops - structure for tracking registered ftrace ops structs
*
* A single ftrace_ops is shared between all enabled replacement functions
* (klp_func structs) which have the same old_addr. This allows the switch
* between function versions to happen instantaneously by updating the klp_ops
* struct's func_stack list. The winner is the klp_func at the top of the
* func_stack (front of the list).
*
* @node: node for the global klp_ops list
* @func_stack: list head for the stack of klp_func's (active func is on top)
* @fops: registered ftrace ops struct
*/
struct klp_ops {
struct list_head node;
struct list_head func_stack;
struct ftrace_ops fops;
};
#include "core.h"
#include "patch.h"
#include "transition.h"
/*
* The klp_mutex protects the global lists and state transitions of any
* structure reachable from them. References to any structure must be obtained
* under mutex protection (except in klp_ftrace_handler(), which uses RCU to
* ensure it gets consistent data).
* klp_mutex is a coarse lock which serializes access to klp data. All
* accesses to klp-related variables and structures must have mutex protection,
* except within the following functions which carefully avoid the need for it:
*
* - klp_ftrace_handler()
* - klp_update_patch_state()
*/
static DEFINE_MUTEX(klp_mutex);
DEFINE_MUTEX(klp_mutex);
static LIST_HEAD(klp_patches);
static LIST_HEAD(klp_ops);
static struct kobject *klp_root_kobj;
static struct klp_ops *klp_find_ops(unsigned long old_addr)
{
struct klp_ops *ops;
struct klp_func *func;
list_for_each_entry(ops, &klp_ops, node) {
func = list_first_entry(&ops->func_stack, struct klp_func,
stack_node);
if (func->old_addr == old_addr)
return ops;
}
return NULL;
}
static bool klp_is_module(struct klp_object *obj)
{
return obj->name;
@@ -117,7 +87,6 @@ static void klp_find_object_module(struct klp_object *obj)
mutex_unlock(&module_mutex);
}
/* klp_mutex must be held by caller */
static bool klp_is_patch_registered(struct klp_patch *patch)
{
struct klp_patch *mypatch;
@@ -314,191 +283,30 @@ static int klp_write_object_relocations(struct module *pmod,
return ret;
}
static void notrace klp_ftrace_handler(unsigned long ip,
unsigned long parent_ip,
struct ftrace_ops *fops,
struct pt_regs *regs)
{
struct klp_ops *ops;
struct klp_func *func;
ops = container_of(fops, struct klp_ops, fops);
rcu_read_lock();
func = list_first_or_null_rcu(&ops->func_stack, struct klp_func,
stack_node);
if (WARN_ON_ONCE(!func))
goto unlock;
klp_arch_set_pc(regs, (unsigned long)func->new_func);
unlock:
rcu_read_unlock();
}
/*
* Convert a function address into the appropriate ftrace location.
*
* Usually this is just the address of the function, but on some architectures
* it's more complicated so allow them to provide a custom behaviour.
*/
#ifndef klp_get_ftrace_location
static unsigned long klp_get_ftrace_location(unsigned long faddr)
{
return faddr;
}
#endif
static void klp_disable_func(struct klp_func *func)
{
struct klp_ops *ops;
if (WARN_ON(func->state != KLP_ENABLED))
return;
if (WARN_ON(!func->old_addr))
return;
ops = klp_find_ops(func->old_addr);
if (WARN_ON(!ops))
return;
if (list_is_singular(&ops->func_stack)) {
unsigned long ftrace_loc;
ftrace_loc = klp_get_ftrace_location(func->old_addr);
if (WARN_ON(!ftrace_loc))
return;
WARN_ON(unregister_ftrace_function(&ops->fops));
WARN_ON(ftrace_set_filter_ip(&ops->fops, ftrace_loc, 1, 0));
list_del_rcu(&func->stack_node);
list_del(&ops->node);
kfree(ops);
} else {
list_del_rcu(&func->stack_node);
}
func->state = KLP_DISABLED;
}
static int klp_enable_func(struct klp_func *func)
{
struct klp_ops *ops;
int ret;
if (WARN_ON(!func->old_addr))
return -EINVAL;
if (WARN_ON(func->state != KLP_DISABLED))
return -EINVAL;
ops = klp_find_ops(func->old_addr);
if (!ops) {
unsigned long ftrace_loc;
ftrace_loc = klp_get_ftrace_location(func->old_addr);
if (!ftrace_loc) {
pr_err("failed to find location for function '%s'\n",
func->old_name);
return -EINVAL;
}
ops = kzalloc(sizeof(*ops), GFP_KERNEL);
if (!ops)
return -ENOMEM;
ops->fops.func = klp_ftrace_handler;
ops->fops.flags = FTRACE_OPS_FL_SAVE_REGS |
FTRACE_OPS_FL_DYNAMIC |
FTRACE_OPS_FL_IPMODIFY;
list_add(&ops->node, &klp_ops);
INIT_LIST_HEAD(&ops->func_stack);
list_add_rcu(&func->stack_node, &ops->func_stack);
ret = ftrace_set_filter_ip(&ops->fops, ftrace_loc, 0, 0);
if (ret) {
pr_err("failed to set ftrace filter for function '%s' (%d)\n",
func->old_name, ret);
goto err;
}
ret = register_ftrace_function(&ops->fops);
if (ret) {
pr_err("failed to register ftrace handler for function '%s' (%d)\n",
func->old_name, ret);
ftrace_set_filter_ip(&ops->fops, ftrace_loc, 1, 0);
goto err;
}
} else {
list_add_rcu(&func->stack_node, &ops->func_stack);
}
func->state = KLP_ENABLED;
return 0;
err:
list_del_rcu(&func->stack_node);
list_del(&ops->node);
kfree(ops);
return ret;
}
static void klp_disable_object(struct klp_object *obj)
{
struct klp_func *func;
klp_for_each_func(obj, func)
if (func->state == KLP_ENABLED)
klp_disable_func(func);
obj->state = KLP_DISABLED;
}
static int klp_enable_object(struct klp_object *obj)
{
struct klp_func *func;
int ret;
if (WARN_ON(obj->state != KLP_DISABLED))
return -EINVAL;
if (WARN_ON(!klp_is_object_loaded(obj)))
return -EINVAL;
klp_for_each_func(obj, func) {
ret = klp_enable_func(func);
if (ret) {
klp_disable_object(obj);
return ret;
}
}
obj->state = KLP_ENABLED;
return 0;
}
static int __klp_disable_patch(struct klp_patch *patch)
{
struct klp_object *obj;
if (klp_transition_patch)
return -EBUSY;
/* enforce stacking: only the last enabled patch can be disabled */
if (!list_is_last(&patch->list, &klp_patches) &&
list_next_entry(patch, list)->state == KLP_ENABLED)
list_next_entry(patch, list)->enabled)
return -EBUSY;
pr_notice("disabling patch '%s'\n", patch->mod->name);
klp_init_transition(patch, KLP_UNPATCHED);
klp_for_each_object(patch, obj) {
if (obj->state == KLP_ENABLED)
klp_disable_object(obj);
}
/*
* Enforce the order of the func->transition writes in
* klp_init_transition() and the TIF_PATCH_PENDING writes in
* klp_start_transition(). In the rare case where klp_ftrace_handler()
* is called shortly after klp_update_patch_state() switches the task,
* this ensures the handler sees that func->transition is set.
*/
smp_wmb();
patch->state = KLP_DISABLED;
klp_start_transition();
klp_try_complete_transition();
patch->enabled = false;
return 0;
}
@@ -522,7 +330,7 @@ int klp_disable_patch(struct klp_patch *patch)
goto err;
}
if (patch->state == KLP_DISABLED) {
if (!patch->enabled) {
ret = -EINVAL;
goto err;
}
@@ -540,32 +348,61 @@ static int __klp_enable_patch(struct klp_patch *patch)
struct klp_object *obj;
int ret;
if (WARN_ON(patch->state != KLP_DISABLED))
if (klp_transition_patch)
return -EBUSY;
if (WARN_ON(patch->enabled))
return -EINVAL;
/* enforce stacking: only the first disabled patch can be enabled */
if (patch->list.prev != &klp_patches &&
list_prev_entry(patch, list)->state == KLP_DISABLED)
!list_prev_entry(patch, list)->enabled)
return -EBUSY;
/*
* A reference is taken on the patch module to prevent it from being
* unloaded.
*
* Note: For immediate (no consistency model) patches we don't allow
* patch modules to unload since there is no safe/sane method to
* determine if a thread is still running in the patched code contained
* in the patch module once the ftrace registration is successful.
*/
if (!try_module_get(patch->mod))
return -ENODEV;
pr_notice("enabling patch '%s'\n", patch->mod->name);
klp_init_transition(patch, KLP_PATCHED);
/*
* Enforce the order of the func->transition writes in
* klp_init_transition() and the ops->func_stack writes in
* klp_patch_object(), so that klp_ftrace_handler() will see the
* func->transition updates before the handler is registered and the
* new funcs become visible to the handler.
*/
smp_wmb();
klp_for_each_object(patch, obj) {
if (!klp_is_object_loaded(obj))
continue;
ret = klp_enable_object(obj);
if (ret)
goto unregister;
ret = klp_patch_object(obj);
if (ret) {
pr_warn("failed to enable patch '%s'\n",
patch->mod->name);
klp_cancel_transition();
return ret;
}
}
patch->state = KLP_ENABLED;
klp_start_transition();
klp_try_complete_transition();
patch->enabled = true;
return 0;
unregister:
WARN_ON(__klp_disable_patch(patch));
return ret;
}
/**
@@ -602,6 +439,7 @@ EXPORT_SYMBOL_GPL(klp_enable_patch);
* /sys/kernel/livepatch
* /sys/kernel/livepatch/<patch>
* /sys/kernel/livepatch/<patch>/enabled
* /sys/kernel/livepatch/<patch>/transition
* /sys/kernel/livepatch/<patch>/<object>
* /sys/kernel/livepatch/<patch>/<object>/<function,sympos>
*/
@@ -611,26 +449,34 @@ static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr,
{
struct klp_patch *patch;
int ret;
unsigned long val;
bool enabled;
ret = kstrtoul(buf, 10, &val);
ret = kstrtobool(buf, &enabled);
if (ret)
return -EINVAL;
if (val != KLP_DISABLED && val != KLP_ENABLED)
return -EINVAL;
return ret;
patch = container_of(kobj, struct klp_patch, kobj);
mutex_lock(&klp_mutex);
if (val == patch->state) {
if (!klp_is_patch_registered(patch)) {
/*
* Module with the patch could either disappear meanwhile or is
* not properly initialized yet.
*/
ret = -EINVAL;
goto err;
}
if (patch->enabled == enabled) {
/* already in requested state */
ret = -EINVAL;
goto err;
}
if (val == KLP_ENABLED) {
if (patch == klp_transition_patch) {
klp_reverse_transition();
} else if (enabled) {
ret = __klp_enable_patch(patch);
if (ret)
goto err;
@@ -655,21 +501,33 @@ static ssize_t enabled_show(struct kobject *kobj,
struct klp_patch *patch;
patch = container_of(kobj, struct klp_patch, kobj);
return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->state);
return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->enabled);
}
static ssize_t transition_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
struct klp_patch *patch;
patch = container_of(kobj, struct klp_patch, kobj);
return snprintf(buf, PAGE_SIZE-1, "%d\n",
patch == klp_transition_patch);
}
static struct kobj_attribute enabled_kobj_attr = __ATTR_RW(enabled);
static struct kobj_attribute transition_kobj_attr = __ATTR_RO(transition);
static struct attribute *klp_patch_attrs[] = {
&enabled_kobj_attr.attr,
&transition_kobj_attr.attr,
NULL
};
static void klp_kobj_release_patch(struct kobject *kobj)
{
/*
* Once we have a consistency model we'll need to module_put() the
* patch module here. See klp_register_patch() for more details.
*/
struct klp_patch *patch;
patch = container_of(kobj, struct klp_patch, kobj);
complete(&patch->finish);
}
static struct kobj_type klp_ktype_patch = {
@@ -740,7 +598,6 @@ static void klp_free_patch(struct klp_patch *patch)
klp_free_objects_limited(patch, NULL);
if (!list_empty(&patch->list))
list_del(&patch->list);
kobject_put(&patch->kobj);
}
static int klp_init_func(struct klp_object *obj, struct klp_func *func)
@@ -749,7 +606,8 @@ static int klp_init_func(struct klp_object *obj, struct klp_func *func)
return -EINVAL;
INIT_LIST_HEAD(&func->stack_node);
func->state = KLP_DISABLED;
func->patched = false;
func->transition = false;
/* The format for the sysfs directory is <function,sympos> where sympos
* is the nth occurrence of this symbol in kallsyms for the patched
@@ -790,6 +648,22 @@ static int klp_init_object_loaded(struct klp_patch *patch,
&func->old_addr);
if (ret)
return ret;
ret = kallsyms_lookup_size_offset(func->old_addr,
&func->old_size, NULL);
if (!ret) {
pr_err("kallsyms size lookup failed for '%s'\n",
func->old_name);
return -ENOENT;
}
ret = kallsyms_lookup_size_offset((unsigned long)func->new_func,
&func->new_size, NULL);
if (!ret) {
pr_err("kallsyms size lookup failed for '%s' replacement\n",
func->old_name);
return -ENOENT;
}
}
return 0;
@@ -804,7 +678,7 @@ static int klp_init_object(struct klp_patch *patch, struct klp_object *obj)
if (!obj->funcs)
return -EINVAL;
obj->state = KLP_DISABLED;
obj->patched = false;
obj->mod = NULL;
klp_find_object_module(obj);
@@ -845,12 +719,15 @@ static int klp_init_patch(struct klp_patch *patch)
mutex_lock(&klp_mutex);
patch->state = KLP_DISABLED;
patch->enabled = false;
init_completion(&patch->finish);
ret = kobject_init_and_add(&patch->kobj, &klp_ktype_patch,
klp_root_kobj, "%s", patch->mod->name);
if (ret)
goto unlock;
if (ret) {
mutex_unlock(&klp_mutex);
return ret;
}
klp_for_each_object(patch, obj) {
ret = klp_init_object(patch, obj);
@@ -866,9 +743,12 @@ static int klp_init_patch(struct klp_patch *patch)
free:
klp_free_objects_limited(patch, obj);
kobject_put(&patch->kobj);
unlock:
mutex_unlock(&klp_mutex);
kobject_put(&patch->kobj);
wait_for_completion(&patch->finish);
return ret;
}
@@ -882,23 +762,29 @@ unlock:
*/
int klp_unregister_patch(struct klp_patch *patch)
{
int ret = 0;
int ret;
mutex_lock(&klp_mutex);
if (!klp_is_patch_registered(patch)) {
ret = -EINVAL;
goto out;
goto err;
}
if (patch->state == KLP_ENABLED) {
if (patch->enabled) {
ret = -EBUSY;
goto out;
goto err;
}
klp_free_patch(patch);
out:
mutex_unlock(&klp_mutex);
kobject_put(&patch->kobj);
wait_for_completion(&patch->finish);
return 0;
err:
mutex_unlock(&klp_mutex);
return ret;
}
@@ -911,12 +797,13 @@ EXPORT_SYMBOL_GPL(klp_unregister_patch);
* Initializes the data structure associated with the patch and
* creates the sysfs interface.
*
* There is no need to take the reference on the patch module here. It is done
* later when the patch is enabled.
*
* Return: 0 on success, otherwise error
*/
int klp_register_patch(struct klp_patch *patch)
{
int ret;
if (!patch || !patch->mod)
return -EINVAL;
@@ -930,20 +817,16 @@ int klp_register_patch(struct klp_patch *patch)
return -ENODEV;
/*
* A reference is taken on the patch module to prevent it from being
* unloaded. Right now, we don't allow patch modules to unload since
* there is currently no method to determine if a thread is still
* running in the patched code contained in the patch module once
* the ftrace registration is successful.
* Architectures without reliable stack traces have to set
* patch->immediate because there's currently no way to patch kthreads
* with the consistency model.
*/
if (!try_module_get(patch->mod))
return -ENODEV;
if (!klp_have_reliable_stack() && !patch->immediate) {
pr_err("This architecture doesn't have support for the livepatch consistency model.\n");
return -ENOSYS;
}
ret = klp_init_patch(patch);
if (ret)
module_put(patch->mod);
return ret;
return klp_init_patch(patch);
}
EXPORT_SYMBOL_GPL(klp_register_patch);
@@ -978,13 +861,17 @@ int klp_module_coming(struct module *mod)
goto err;
}
if (patch->state == KLP_DISABLED)
/*
* Only patch the module if the patch is enabled or is
* in transition.
*/
if (!patch->enabled && patch != klp_transition_patch)
break;
pr_notice("applying patch '%s' to loading module '%s'\n",
patch->mod->name, obj->mod->name);
ret = klp_enable_object(obj);
ret = klp_patch_object(obj);
if (ret) {
pr_warn("failed to apply patch '%s' to module '%s' (%d)\n",
patch->mod->name, obj->mod->name, ret);
@@ -1035,10 +922,14 @@ void klp_module_going(struct module *mod)
if (!klp_is_module(obj) || strcmp(obj->name, mod->name))
continue;
if (patch->state != KLP_DISABLED) {
/*
* Only unpatch the module if the patch is enabled or
* is in transition.
*/
if (patch->enabled || patch == klp_transition_patch) {
pr_notice("reverting patch '%s' on unloading module '%s'\n",
patch->mod->name, obj->mod->name);
klp_disable_object(obj);
klp_unpatch_object(obj);
}
klp_free_object_loaded(obj);
kernel/livepatch/core.h
+6
View File
@@ -0,0 +1,6 @@
#ifndef _LIVEPATCH_CORE_H
#define _LIVEPATCH_CORE_H
extern struct mutex klp_mutex;
#endif /* _LIVEPATCH_CORE_H */
kernel/livepatch/patch.c
+272
View File
@@ -0,0 +1,272 @@
/*
* patch.c - livepatch patching functions
*
* Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
* Copyright (C) 2014 SUSE
* Copyright (C) 2015 Josh Poimboeuf <jpoimboe@redhat.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/livepatch.h>
#include <linux/list.h>
#include <linux/ftrace.h>
#include <linux/rculist.h>
#include <linux/slab.h>
#include <linux/bug.h>
#include <linux/printk.h>
#include "patch.h"
#include "transition.h"
static LIST_HEAD(klp_ops);
struct klp_ops *klp_find_ops(unsigned long old_addr)
{
struct klp_ops *ops;
struct klp_func *func;
list_for_each_entry(ops, &klp_ops, node) {
func = list_first_entry(&ops->func_stack, struct klp_func,
stack_node);
if (func->old_addr == old_addr)
return ops;
}
return NULL;
}
static void notrace klp_ftrace_handler(unsigned long ip,
unsigned long parent_ip,
struct ftrace_ops *fops,
struct pt_regs *regs)
{
struct klp_ops *ops;
struct klp_func *func;
int patch_state;
ops = container_of(fops, struct klp_ops, fops);
rcu_read_lock();
func = list_first_or_null_rcu(&ops->func_stack, struct klp_func,
stack_node);
/*
* func should never be NULL because preemption should be disabled here
* and unregister_ftrace_function() does the equivalent of a
* synchronize_sched() before the func_stack removal.
*/
if (WARN_ON_ONCE(!func))
goto unlock;
/*
* In the enable path, enforce the order of the ops->func_stack and
* func->transition reads. The corresponding write barrier is in
* __klp_enable_patch().
*
* (Note that this barrier technically isn't needed in the disable
* path. In the rare case where klp_update_patch_state() runs before
* this handler, its TIF_PATCH_PENDING read and this func->transition
* read need to be ordered. But klp_update_patch_state() already
* enforces that.)
*/
smp_rmb();
if (unlikely(func->transition)) {
/*
* Enforce the order of the func->transition and
* current->patch_state reads. Otherwise we could read an
* out-of-date task state and pick the wrong function. The
* corresponding write barrier is in klp_init_transition().
*/
smp_rmb();
patch_state = current->patch_state;
WARN_ON_ONCE(patch_state == KLP_UNDEFINED);
if (patch_state == KLP_UNPATCHED) {
/*
* Use the previously patched version of the function.
* If no previous patches exist, continue with the
* original function.
*/
func = list_entry_rcu(func->stack_node.next,
struct klp_func, stack_node);
if (&func->stack_node == &ops->func_stack)
goto unlock;
}
}
klp_arch_set_pc(regs, (unsigned long)func->new_func);
unlock:
rcu_read_unlock();
}
/*
* Convert a function address into the appropriate ftrace location.
*
* Usually this is just the address of the function, but on some architectures
* it's more complicated so allow them to provide a custom behaviour.
*/
#ifndef klp_get_ftrace_location
static unsigned long klp_get_ftrace_location(unsigned long faddr)
{
return faddr;
}
#endif
static void klp_unpatch_func(struct klp_func *func)
{
struct klp_ops *ops;
if (WARN_ON(!func->patched))
return;
if (WARN_ON(!func->old_addr))
return;
ops = klp_find_ops(func->old_addr);
if (WARN_ON(!ops))
return;
if (list_is_singular(&ops->func_stack)) {
unsigned long ftrace_loc;
ftrace_loc = klp_get_ftrace_location(func->old_addr);
if (WARN_ON(!ftrace_loc))
return;
WARN_ON(unregister_ftrace_function(&ops->fops));
WARN_ON(ftrace_set_filter_ip(&ops->fops, ftrace_loc, 1, 0));
list_del_rcu(&func->stack_node);
list_del(&ops->node);
kfree(ops);
} else {
list_del_rcu(&func->stack_node);
}
func->patched = false;
}
static int klp_patch_func(struct klp_func *func)
{
struct klp_ops *ops;
int ret;
if (WARN_ON(!func->old_addr))
return -EINVAL;
if (WARN_ON(func->patched))
return -EINVAL;
ops = klp_find_ops(func->old_addr);
if (!ops) {
unsigned long ftrace_loc;
ftrace_loc = klp_get_ftrace_location(func->old_addr);
if (!ftrace_loc) {
pr_err("failed to find location for function '%s'\n",
func->old_name);
return -EINVAL;
}
ops = kzalloc(sizeof(*ops), GFP_KERNEL);
if (!ops)
return -ENOMEM;
ops->fops.func = klp_ftrace_handler;
ops->fops.flags = FTRACE_OPS_FL_SAVE_REGS |
FTRACE_OPS_FL_DYNAMIC |
FTRACE_OPS_FL_IPMODIFY;
list_add(&ops->node, &klp_ops);
INIT_LIST_HEAD(&ops->func_stack);
list_add_rcu(&func->stack_node, &ops->func_stack);
ret = ftrace_set_filter_ip(&ops->fops, ftrace_loc, 0, 0);
if (ret) {
pr_err("failed to set ftrace filter for function '%s' (%d)\n",
func->old_name, ret);
goto err;
}
ret = register_ftrace_function(&ops->fops);
if (ret) {
pr_err("failed to register ftrace handler for function '%s' (%d)\n",
func->old_name, ret);
ftrace_set_filter_ip(&ops->fops, ftrace_loc, 1, 0);
goto err;
}
} else {
list_add_rcu(&func->stack_node, &ops->func_stack);
}
func->patched = true;
return 0;
err:
list_del_rcu(&func->stack_node);
list_del(&ops->node);
kfree(ops);
return ret;
}
void klp_unpatch_object(struct klp_object *obj)
{
struct klp_func *func;
klp_for_each_func(obj, func)
if (func->patched)
klp_unpatch_func(func);
obj->patched = false;
}
int klp_patch_object(struct klp_object *obj)
{
struct klp_func *func;
int ret;
if (WARN_ON(obj->patched))
return -EINVAL;
klp_for_each_func(obj, func) {
ret = klp_patch_func(func);
if (ret) {
klp_unpatch_object(obj);
return ret;
}
}
obj->patched = true;
return 0;
}
void klp_unpatch_objects(struct klp_patch *patch)
{
struct klp_object *obj;
klp_for_each_object(patch, obj)
if (obj->patched)
klp_unpatch_object(obj);
}
kernel/livepatch/patch.h
+33
View File
@@ -0,0 +1,33 @@
#ifndef _LIVEPATCH_PATCH_H
#define _LIVEPATCH_PATCH_H
#include <linux/livepatch.h>
#include <linux/list.h>
#include <linux/ftrace.h>
/**
* struct klp_ops - structure for tracking registered ftrace ops structs
*
* A single ftrace_ops is shared between all enabled replacement functions
* (klp_func structs) which have the same old_addr. This allows the switch
* between function versions to happen instantaneously by updating the klp_ops
* struct's func_stack list. The winner is the klp_func at the top of the
* func_stack (front of the list).
*
* @node: node for the global klp_ops list
* @func_stack: list head for the stack of klp_func's (active func is on top)
* @fops: registered ftrace ops struct
*/
struct klp_ops {
struct list_head node;
struct list_head func_stack;
struct ftrace_ops fops;
};
struct klp_ops *klp_find_ops(unsigned long old_addr);
int klp_patch_object(struct klp_object *obj);
void klp_unpatch_object(struct klp_object *obj);
void klp_unpatch_objects(struct klp_patch *patch);
#endif /* _LIVEPATCH_PATCH_H */
kernel/livepatch/transition.c
+553
View File
@@ -0,0 +1,553 @@
/*
* transition.c - Kernel Live Patching transition functions
*
* Copyright (C) 2015-2016 Josh Poimboeuf <jpoimboe@redhat.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/cpu.h>
#include <linux/stacktrace.h>
#include "core.h"
#include "patch.h"
#include "transition.h"
#include "../sched/sched.h"
#define MAX_STACK_ENTRIES 100
#define STACK_ERR_BUF_SIZE 128
struct klp_patch *klp_transition_patch;
static int klp_target_state = KLP_UNDEFINED;
/*
* This work can be performed periodically to finish patching or unpatching any
* "straggler" tasks which failed to transition in the first attempt.
*/
static void klp_transition_work_fn(struct work_struct *work)
{
mutex_lock(&klp_mutex);
if (klp_transition_patch)
klp_try_complete_transition();
mutex_unlock(&klp_mutex);
}
static DECLARE_DELAYED_WORK(klp_transition_work, klp_transition_work_fn);
/*
* The transition to the target patch state is complete. Clean up the data
* structures.
*/
static void klp_complete_transition(void)
{
struct klp_object *obj;
struct klp_func *func;
struct task_struct *g, *task;
unsigned int cpu;
bool immediate_func = false;
if (klp_target_state == KLP_UNPATCHED) {
/*
* All tasks have transitioned to KLP_UNPATCHED so we can now
* remove the new functions from the func_stack.
*/
klp_unpatch_objects(klp_transition_patch);
/*
* Make sure klp_ftrace_handler() can no longer see functions
* from this patch on the ops->func_stack. Otherwise, after
* func->transition gets cleared, the handler may choose a
* removed function.
*/
synchronize_rcu();
}
if (klp_transition_patch->immediate)
goto done;
klp_for_each_object(klp_transition_patch, obj) {
klp_for_each_func(obj, func) {
func->transition = false;
if (func->immediate)
immediate_func = true;
}
}
if (klp_target_state == KLP_UNPATCHED && !immediate_func)
module_put(klp_transition_patch->mod);
/* Prevent klp_ftrace_handler() from seeing KLP_UNDEFINED state */
if (klp_target_state == KLP_PATCHED)
synchronize_rcu();
read_lock(&tasklist_lock);
for_each_process_thread(g, task) {
WARN_ON_ONCE(test_tsk_thread_flag(task, TIF_PATCH_PENDING));
task->patch_state = KLP_UNDEFINED;
}
read_unlock(&tasklist_lock);
for_each_possible_cpu(cpu) {
task = idle_task(cpu);
WARN_ON_ONCE(test_tsk_thread_flag(task, TIF_PATCH_PENDING));
task->patch_state = KLP_UNDEFINED;
}
done:
klp_target_state = KLP_UNDEFINED;
klp_transition_patch = NULL;
}
/*
* This is called in the error path, to cancel a transition before it has
* started, i.e. klp_init_transition() has been called but
* klp_start_transition() hasn't. If the transition *has* been started,
* klp_reverse_transition() should be used instead.
*/
void klp_cancel_transition(void)
{
if (WARN_ON_ONCE(klp_target_state != KLP_PATCHED))
return;
klp_target_state = KLP_UNPATCHED;
klp_complete_transition();
}
/*
* Switch the patched state of the task to the set of functions in the target
* patch state.
*
* NOTE: If task is not 'current', the caller must ensure the task is inactive.
* Otherwise klp_ftrace_handler() might read the wrong 'patch_state' value.
*/
void klp_update_patch_state(struct task_struct *task)
{
rcu_read_lock();
/*
* This test_and_clear_tsk_thread_flag() call also serves as a read
* barrier (smp_rmb) for two cases:
*
* 1) Enforce the order of the TIF_PATCH_PENDING read and the
* klp_target_state read. The corresponding write barrier is in
* klp_init_transition().
*
* 2) Enforce the order of the TIF_PATCH_PENDING read and a future read
* of func->transition, if klp_ftrace_handler() is called later on
* the same CPU. See __klp_disable_patch().
*/
if (test_and_clear_tsk_thread_flag(task, TIF_PATCH_PENDING))
task->patch_state = READ_ONCE(klp_target_state);
rcu_read_unlock();
}
/*
* Determine whether the given stack trace includes any references to a
* to-be-patched or to-be-unpatched function.
*/
static int klp_check_stack_func(struct klp_func *func,
struct stack_trace *trace)
{
unsigned long func_addr, func_size, address;
struct klp_ops *ops;
int i;
if (func->immediate)
return 0;
for (i = 0; i < trace->nr_entries; i++) {
address = trace->entries[i];
if (klp_target_state == KLP_UNPATCHED) {
/*
* Check for the to-be-unpatched function
* (the func itself).
*/
func_addr = (unsigned long)func->new_func;
func_size = func->new_size;
} else {
/*
* Check for the to-be-patched function
* (the previous func).
*/
ops = klp_find_ops(func->old_addr);
if (list_is_singular(&ops->func_stack)) {
/* original function */
func_addr = func->old_addr;
func_size = func->old_size;
} else {
/* previously patched function */
struct klp_func *prev;
prev = list_next_entry(func, stack_node);
func_addr = (unsigned long)prev->new_func;
func_size = prev->new_size;
}
}
if (address >= func_addr && address < func_addr + func_size)
return -EAGAIN;
}
return 0;
}
/*
* Determine whether it's safe to transition the task to the target patch state
* by looking for any to-be-patched or to-be-unpatched functions on its stack.
*/
static int klp_check_stack(struct task_struct *task, char *err_buf)
{
static unsigned long entries[MAX_STACK_ENTRIES];
struct stack_trace trace;
struct klp_object *obj;
struct klp_func *func;
int ret;
trace.skip = 0;
trace.nr_entries = 0;
trace.max_entries = MAX_STACK_ENTRIES;
trace.entries = entries;
ret = save_stack_trace_tsk_reliable(task, &trace);
WARN_ON_ONCE(ret == -ENOSYS);
if (ret) {
snprintf(err_buf, STACK_ERR_BUF_SIZE,
"%s: %s:%d has an unreliable stack\n",
__func__, task->comm, task->pid);
return ret;
}
klp_for_each_object(klp_transition_patch, obj) {
if (!obj->patched)
continue;
klp_for_each_func(obj, func) {
ret = klp_check_stack_func(func, &trace);
if (ret) {
snprintf(err_buf, STACK_ERR_BUF_SIZE,
"%s: %s:%d is sleeping on function %s\n",
__func__, task->comm, task->pid,
func->old_name);
return ret;
}
}
}
return 0;
}
/*
* Try to safely switch a task to the target patch state. If it's currently
* running, or it's sleeping on a to-be-patched or to-be-unpatched function, or
* if the stack is unreliable, return false.
*/
static bool klp_try_switch_task(struct task_struct *task)
{
struct rq *rq;
struct rq_flags flags;
int ret;
bool success = false;
char err_buf[STACK_ERR_BUF_SIZE];
err_buf[0] = '\0';
/* check if this task has already switched over */
if (task->patch_state == klp_target_state)
return true;
/*
* For arches which don't have reliable stack traces, we have to rely
* on other methods (e.g., switching tasks at kernel exit).
*/
if (!klp_have_reliable_stack())
return false;
/*
* Now try to check the stack for any to-be-patched or to-be-unpatched
* functions. If all goes well, switch the task to the target patch
* state.
*/
rq = task_rq_lock(task, &flags);
if (task_running(rq, task) && task != current) {
snprintf(err_buf, STACK_ERR_BUF_SIZE,
"%s: %s:%d is running\n", __func__, task->comm,
task->pid);
goto done;
}
ret = klp_check_stack(task, err_buf);
if (ret)
goto done;
success = true;
clear_tsk_thread_flag(task, TIF_PATCH_PENDING);
task->patch_state = klp_target_state;
done:
task_rq_unlock(rq, task, &flags);
/*
* Due to console deadlock issues, pr_debug() can't be used while
* holding the task rq lock. Instead we have to use a temporary buffer
* and print the debug message after releasing the lock.
*/
if (err_buf[0] != '\0')
pr_debug("%s", err_buf);
return success;
}
/*
* Try to switch all remaining tasks to the target patch state by walking the
* stacks of sleeping tasks and looking for any to-be-patched or
* to-be-unpatched functions. If such functions are found, the task can't be
* switched yet.
*
* If any tasks are still stuck in the initial patch state, schedule a retry.
*/
void klp_try_complete_transition(void)
{
unsigned int cpu;
struct task_struct *g, *task;
bool complete = true;
WARN_ON_ONCE(klp_target_state == KLP_UNDEFINED);
/*
* If the patch can be applied or reverted immediately, skip the
* per-task transitions.
*/
if (klp_transition_patch->immediate)
goto success;
/*
* Try to switch the tasks to the target patch state by walking their
* stacks and looking for any to-be-patched or to-be-unpatched
* functions. If such functions are found on a stack, or if the stack
* is deemed unreliable, the task can't be switched yet.
*
* Usually this will transition most (or all) of the tasks on a system
* unless the patch includes changes to a very common function.
*/
read_lock(&tasklist_lock);
for_each_process_thread(g, task)
if (!klp_try_switch_task(task))
complete = false;
read_unlock(&tasklist_lock);
/*
* Ditto for the idle "swapper" tasks.
*/
get_online_cpus();
for_each_possible_cpu(cpu) {
task = idle_task(cpu);
if (cpu_online(cpu)) {
if (!klp_try_switch_task(task))
complete = false;
} else if (task->patch_state != klp_target_state) {
/* offline idle tasks can be switched immediately */
clear_tsk_thread_flag(task, TIF_PATCH_PENDING);
task->patch_state = klp_target_state;
}
}
put_online_cpus();
if (!complete) {
/*
* Some tasks weren't able to be switched over. Try again
* later and/or wait for other methods like kernel exit
* switching.
*/
schedule_delayed_work(&klp_transition_work,
round_jiffies_relative(HZ));
return;
}
success:
pr_notice("'%s': %s complete\n", klp_transition_patch->mod->name,
klp_target_state == KLP_PATCHED ? "patching" : "unpatching");
/* we're done, now cleanup the data structures */
klp_complete_transition();
}
/*
* Start the transition to the specified target patch state so tasks can begin
* switching to it.
*/
void klp_start_transition(void)
{
struct task_struct *g, *task;
unsigned int cpu;
WARN_ON_ONCE(klp_target_state == KLP_UNDEFINED);
pr_notice("'%s': %s...\n", klp_transition_patch->mod->name,
klp_target_state == KLP_PATCHED ? "patching" : "unpatching");
/*
* If the patch can be applied or reverted immediately, skip the
* per-task transitions.
*/
if (klp_transition_patch->immediate)
return;
/*
* Mark all normal tasks as needing a patch state update. They'll
* switch either in klp_try_complete_transition() or as they exit the
* kernel.
*/
read_lock(&tasklist_lock);
for_each_process_thread(g, task)
if (task->patch_state != klp_target_state)
set_tsk_thread_flag(task, TIF_PATCH_PENDING);
read_unlock(&tasklist_lock);
/*
* Mark all idle tasks as needing a patch state update. They'll switch
* either in klp_try_complete_transition() or at the idle loop switch
* point.
*/
for_each_possible_cpu(cpu) {
task = idle_task(cpu);
if (task->patch_state != klp_target_state)
set_tsk_thread_flag(task, TIF_PATCH_PENDING);
}
}
/*
* Initialize the global target patch state and all tasks to the initial patch
* state, and initialize all function transition states to true in preparation
* for patching or unpatching.
*/
void klp_init_transition(struct klp_patch *patch, int state)
{
struct task_struct *g, *task;
unsigned int cpu;
struct klp_object *obj;
struct klp_func *func;
int initial_state = !state;
WARN_ON_ONCE(klp_target_state != KLP_UNDEFINED);
klp_transition_patch = patch;
/*
* Set the global target patch state which tasks will switch to. This
* has no effect until the TIF_PATCH_PENDING flags get set later.
*/
klp_target_state = state;
/*
* If the patch can be applied or reverted immediately, skip the
* per-task transitions.
*/
if (patch->immediate)
return;
/*
* Initialize all tasks to the initial patch state to prepare them for
* switching to the target state.
*/
read_lock(&tasklist_lock);
for_each_process_thread(g, task) {
WARN_ON_ONCE(task->patch_state != KLP_UNDEFINED);
task->patch_state = initial_state;
}
read_unlock(&tasklist_lock);
/*
* Ditto for the idle "swapper" tasks.
*/
for_each_possible_cpu(cpu) {
task = idle_task(cpu);
WARN_ON_ONCE(task->patch_state != KLP_UNDEFINED);
task->patch_state = initial_state;
}
/*
* Enforce the order of the task->patch_state initializations and the
* func->transition updates to ensure that klp_ftrace_handler() doesn't
* see a func in transition with a task->patch_state of KLP_UNDEFINED.
*
* Also enforce the order of the klp_target_state write and future
* TIF_PATCH_PENDING writes to ensure klp_update_patch_state() doesn't
* set a task->patch_state to KLP_UNDEFINED.
*/
smp_wmb();
/*
* Set the func transition states so klp_ftrace_handler() will know to
* switch to the transition logic.
*
* When patching, the funcs aren't yet in the func_stack and will be
* made visible to the ftrace handler shortly by the calls to
* klp_patch_object().
*
* When unpatching, the funcs are already in the func_stack and so are
* already visible to the ftrace handler.
*/
klp_for_each_object(patch, obj)
klp_for_each_func(obj, func)
func->transition = true;
}
/*
* This function can be called in the middle of an existing transition to
* reverse the direction of the target patch state. This can be done to
* effectively cancel an existing enable or disable operation if there are any
* tasks which are stuck in the initial patch state.
*/
void klp_reverse_transition(void)
{
unsigned int cpu;
struct task_struct *g, *task;
klp_transition_patch->enabled = !klp_transition_patch->enabled;
klp_target_state = !klp_target_state;
/*
* Clear all TIF_PATCH_PENDING flags to prevent races caused by
* klp_update_patch_state() running in parallel with
* klp_start_transition().
*/
read_lock(&tasklist_lock);
for_each_process_thread(g, task)
clear_tsk_thread_flag(task, TIF_PATCH_PENDING);
read_unlock(&tasklist_lock);
for_each_possible_cpu(cpu)
clear_tsk_thread_flag(idle_task(cpu), TIF_PATCH_PENDING);
/* Let any remaining calls to klp_update_patch_state() complete */
synchronize_rcu();
klp_start_transition();
}
/* Called from copy_process() during fork */
void klp_copy_process(struct task_struct *child)
{
child->patch_state = current->patch_state;
/* TIF_PATCH_PENDING gets copied in setup_thread_stack() */
}
kernel/livepatch/transition.h
+14
View File
@@ -0,0 +1,14 @@
#ifndef _LIVEPATCH_TRANSITION_H
#define _LIVEPATCH_TRANSITION_H
#include <linux/livepatch.h>
extern struct klp_patch *klp_transition_patch;
void klp_init_transition(struct klp_patch *patch, int state);
void klp_cancel_transition(void);
void klp_start_transition(void);
void klp_try_complete_transition(void);
void klp_reverse_transition(void);
#endif /* _LIVEPATCH_TRANSITION_H */