ANDROID: signal: Add vendor hook for memory reap

Add a vendor hook to determine if the memory of a process that received
a SIGKILL can be reaped. This is a partial cherry-pick of aosp/1724512
and aosp/2093626.

Bug: 232062955
Change-Id: I75072bd264df33caff67d083821ee6f33ca83af9
Signed-off-by: Tangquan Zheng <zhengtangquan@oppo.com>
[cmllamas: minor cleanups and refactoring]
Signed-off-by: Carlos Llamas <cmllamas@google.com>

drivers/android/vendor_hooks.c

@@ -65,6 +65,7 @@
  * associated with them) to allow external modules to probe them.
  */
 EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_do_send_sig_info);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_killed_process);
 EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_arch_set_freq_scale);
 EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_transaction_init);
 EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_set_priority);

include/linux/oom.h

@@ -114,4 +114,5 @@ extern void dump_tasks(struct oom_control *oc);
 
 extern struct task_struct *find_lock_task_mm(struct task_struct *p);
 
+void add_to_oom_reaper(struct task_struct *p);
 #endif /* _INCLUDE_LINUX_OOM_H */

include/trace/hooks/signal.h

@@ -11,6 +11,9 @@ struct task_struct;
 DECLARE_HOOK(android_vh_do_send_sig_info,
 	TP_PROTO(int sig, struct task_struct *killer, struct task_struct *dst),
 	TP_ARGS(sig, killer, dst));
+DECLARE_HOOK(android_vh_killed_process,
+	TP_PROTO(struct task_struct *killer, struct task_struct *dst, bool *reap),
+	TP_ARGS(killer, dst, reap));
 #endif /* _TRACE_HOOK_SIGNAL_H */
 /* This part must be outside protection */
 #include <trace/define_trace.h>

kernel/signal.c

@@ -47,6 +47,7 @@
 #include <linux/cgroup.h>
 #include <linux/audit.h>
 #include <linux/sysctl.h>
+#include <linux/oom.h>
 #include <uapi/linux/pidfd.h>
 
 #define CREATE_TRACE_POINTS
@@ -1450,8 +1451,16 @@ int group_send_sig_info(int sig, struct kernel_siginfo *info,
 	ret = check_kill_permission(sig, info, p);
 	rcu_read_unlock();
 
-	if (!ret && sig)
+	if (!ret && sig) {
 		ret = do_send_sig_info(sig, info, p, type);
+		if (!ret && sig == SIGKILL) {
+			bool reap = false;
+
+			trace_android_vh_killed_process(current, p, &reap);
+			if (reap)
+				add_to_oom_reaper(p);
+		}
+	}
 
 	return ret;
 }

mm/oom_kill.c

@@ -917,6 +917,21 @@ static bool task_will_free_mem(struct task_struct *task)
 	return ret;
 }
 
+/* Adds a killed process to the reaper. @p->mm has to be non NULL. */
+void add_to_oom_reaper(struct task_struct *p)
+{
+	p = find_lock_task_mm(p);
+	if (!p)
+		return;
+
+	if (task_will_free_mem(p)) {
+		if (!cmpxchg(&p->signal->oom_mm, NULL, p->mm))
+			mmgrab(p->signal->oom_mm);
+		queue_oom_reaper(p);
+	}
+	task_unlock(p);
+}
+
 static void __oom_kill_process(struct task_struct *victim, const char *message)
 {
 	struct task_struct *p;